code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/StatusAgent.cs

   1 #region\r
   2 /* -----------------------------------------------------------------------\r
   3  * <copyright file="StatusAgent.cs" company="GRNet">\r
   4  * \r
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
   6  *\r
   7  * Redistribution and use in source and binary forms, with or\r
   8  * without modification, are permitted provided that the following\r
   9  * conditions are met:\r
  10  *\r
  11  *   1. Redistributions of source code must retain the above\r
  12  *      copyright notice, this list of conditions and the following\r
  13  *      disclaimer.\r
  14  *\r
  15  *   2. Redistributions in binary form must reproduce the above\r
  16  *      copyright notice, this list of conditions and the following\r
  17  *      disclaimer in the documentation and/or other materials\r
  18  *      provided with the distribution.\r
  19  *\r
  20  *\r
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
  32  * POSSIBILITY OF SUCH DAMAGE.\r
  33  *\r
  34  * The views and conclusions contained in the software and\r
  35  * documentation are those of the authors and should not be\r
  36  * interpreted as representing official policies, either expressed\r
  37  * or implied, of GRNET S.A.\r
  38  * </copyright>\r
  39  * -----------------------------------------------------------------------\r
  40  */\r
  41 #endregion\r
  42 using System;\r
  43 using System.Collections.Generic;\r
  44 using System.ComponentModel.Composition;\r
  45 using System.Data;\r
  46 using System.Data.SQLite;\r
  47 using System.Diagnostics;\r
  48 using System.Diagnostics.Contracts;\r
  49 using System.IO;\r
  50 using System.Linq;\r
  51 using System.Reflection;\r
  52 using System.Security.Cryptography;\r
  53 using System.Text;\r
  54 using System.Threading;\r
  55 using System.Threading.Tasks;\r
  56 using Castle.ActiveRecord;\r
  57 using Castle.ActiveRecord.Framework;\r
  58 using Castle.ActiveRecord.Framework.Config;\r
  59 using Castle.ActiveRecord.Queries;\r
  60 using NHibernate;\r
  61 using NHibernate.ByteCode.Castle;\r
  62 using NHibernate.Cfg;\r
  63 using NHibernate.Cfg.Loquacious;\r
  64 using NHibernate.Dialect;\r
  65 using NHibernate.Exceptions;\r
  66 using Pithos.Interfaces;\r
  67 using Pithos.Network;\r
  68 using log4net;\r
  69 using Environment = System.Environment;\r
  70 \r
  71 namespace Pithos.Core.Agents\r
  72 {\r
  73     [Export(typeof(IStatusChecker)),Export(typeof(IStatusKeeper))]\r
  74     public class StatusAgent:IStatusChecker,IStatusKeeper\r
  75     {\r
  76         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  77 \r
  78         [System.ComponentModel.Composition.Import]\r
  79         public IPithosSettings Settings { get; set; }\r
  80 \r
  81         [System.ComponentModel.Composition.Import]\r
  82         public IStatusNotification StatusNotification { get; set; }\r
  83 \r
  84         private Agent<Action> _persistenceAgent;\r
  85 \r
  86 \r
  87 \r
  88         public StatusAgent()\r
  89         {            \r
  90             var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);\r
  91 \r
  92             _pithosDataPath = Path.Combine(appDataPath , "GRNET\\PITHOS");\r
  93             if (!Directory.Exists(_pithosDataPath))\r
  94                 Directory.CreateDirectory(_pithosDataPath);\r
  95 \r
  96             var dbPath = Path.Combine(_pithosDataPath, "pithos.db");\r
  97 \r
  98             MigrateOldDb(dbPath, appDataPath);\r
  99 \r
 100 \r
 101             var source = GetConfiguration(_pithosDataPath);\r
 102             ActiveRecordStarter.Initialize(source,typeof(FileState),typeof(FileTag));\r
 103 \r
 104             UpgradeDatabase();\r
 105 \r
 106 \r
 107 \r
 108             if (!File.Exists(dbPath))\r
 109                 ActiveRecordStarter.CreateSchema();\r
 110 \r
 111             CreateTrigger();\r
 112 \r
 113         }\r
 114 \r
 115 \r
 116         private static void MigrateOldDb(string dbPath, string appDataPath)\r
 117         {\r
 118             if(String.IsNullOrWhiteSpace(dbPath))\r
 119                 throw new ArgumentNullException("dbPath");\r
 120             if(String.IsNullOrWhiteSpace(appDataPath))\r
 121                 throw new ArgumentNullException("appDataPath");\r
 122             Contract.EndContractBlock();\r
 123 \r
 124             var oldDbPath = Path.Combine(appDataPath, "Pithos", "pithos.db");\r
 125             var oldDbInfo = new FileInfo(oldDbPath);\r
 126             if (oldDbInfo.Exists && !File.Exists(dbPath))\r
 127             {\r
 128                 Log.InfoFormat("Moving database from {0} to {1}",oldDbInfo.FullName,dbPath);\r
 129                 var oldDirectory = oldDbInfo.Directory;\r
 130                 oldDbInfo.MoveTo(dbPath);\r
 131                 \r
 132                 if (Log.IsDebugEnabled)\r
 133                     Log.DebugFormat("Deleting {0}",oldDirectory.FullName);\r
 134                 \r
 135                 oldDirectory.Delete(true);\r
 136             }\r
 137         }\r
 138 \r
 139         private T? GetNull<T>(string commandText,SQLiteConnection connection) where T:struct \r
 140         {\r
 141             using (var command= new SQLiteCommand(commandText, connection))\r
 142             {\r
 143                 var result = command.ExecuteScalar();\r
 144                 if (result == null)\r
 145                     return null;\r
 146                 return (T)result;\r
 147             }\r
 148         }\r
 149 \r
 150         private T Get<T>(string commandText,SQLiteConnection connection) \r
 151         {\r
 152             using (var command= new SQLiteCommand(commandText, connection))\r
 153             {\r
 154                 var result = command.ExecuteScalar();\r
 155                 if (result == null)\r
 156                     return default(T);\r
 157                 return (T)result;\r
 158             }\r
 159         }\r
 160 \r
 161         private int Run(string commandText,SQLiteConnection connection)\r
 162         {\r
 163             using (var command= new SQLiteCommand(commandText, connection))\r
 164             {\r
 165                 var result=command.ExecuteNonQuery();\r
 166                 return result;\r
 167             }\r
 168         }\r
 169 \r
 170         private void UpgradeDatabase()\r
 171         {\r
 172             const string hasVersionText = "select 1 from sqlite_master where name='Version'";\r
 173 \r
 174             const string hasFilestateText = "select 1 from sqlite_master where name='FileState'";\r
 175 \r
 176             const string getVersionCmd = "select Version from version where Id=1";\r
 177 \r
 178             const string createVersionCmd = "create table Version(Id integer,Version TEXT);\n" +\r
 179                                             "INSERT INTO VERSION (Id,Version) VALUES(1,'0.0.0.0');";\r
 180             const string createFileStateCmd =\r
 181                 "CREATE TABLE FileState (Id UNIQUEIDENTIFIER not null, ObjectID TEXT COLLATE NOCASE, FilePath TEXT unique COLLATE NOCASE, OverlayStatus INTEGER, FileStatus INTEGER, ConflictReason TEXT, Checksum TEXT COLLATE NOCASE, ETag TEXT not null COLLATE NOCASE, LastMD5 TEXT not null COLLATE NOCASE,LastWriteDate DATETIME, LastLength INTEGER, Version INTEGER, VersionTimeStamp DATETIME, IsShared INTEGER, SharedBy TEXT, ShareWrite INTEGER, IsFolder INTEGER, Modified DATETIME,Hashes TEXT null COLLATE NOCASE, primary key (Id),unique (FilePath))";\r
 182             const string upgradeText = "PRAGMA writable_schema = 1;\n" +\r
 183                                    "UPDATE SQLITE_MASTER SET SQL = 'CREATE TABLE FileState (Id UNIQUEIDENTIFIER not null, ObjectID TEXT COLLATE NOCASE, FilePath TEXT unique COLLATE NOCASE, OverlayStatus INTEGER, FileStatus INTEGER, ConflictReason TEXT, Checksum TEXT COLLATE NOCASE, ETag TEXT not null COLLATE NOCASE, LastMD5 TEXT not null COLLATE NOCASE,LastWriteDate DATETIME, LastLength INTEGER, Version INTEGER, VersionTimeStamp DATETIME, IsShared INTEGER, SharedBy TEXT, ShareWrite INTEGER, IsFolder INTEGER, Modified DATETIME, Hashes TEXT null COLLATE NOCASE, primary key (Id),unique (FilePath))' WHERE NAME = 'FileState';\n" +\r
 184                                    "PRAGMA writable_schema = 0;\n" +\r
 185                                    "VACUUM;";\r
 186 \r
 187             using (var connection = GetConnection())\r
 188             {\r
 189                 var hasVersion = false;\r
 190                 hasVersion = GetNull<long>(hasVersionText, connection).HasValue;\r
 191 \r
 192                 var storedVersion = new Version();\r
 193 \r
 194                 if (hasVersion)\r
 195                 {\r
 196                     var versionTxt = Get<string>(getVersionCmd, connection);\r
 197                     storedVersion = new Version(versionTxt);\r
 198                 }\r
 199                 else\r
 200                     Run(createVersionCmd, connection);\r
 201 \r
 202                 var hasFileState = false;\r
 203                 hasFileState = GetNull<long>(hasFilestateText, connection).HasValue;\r
 204                 if (!hasFileState)\r
 205                 {\r
 206                     Run(createFileStateCmd, connection);\r
 207                 }\r
 208 \r
 209                 var actualVersion = Assembly.GetEntryAssembly().GetName().Version;\r
 210                 if (!hasVersion || actualVersion > storedVersion)\r
 211                     Run(upgradeText, connection);\r
 212 \r
 213                 if (actualVersion != storedVersion)\r
 214                     using (var updateVersionCmd = new SQLiteCommand("UPDATE VERSION SET Version=:version where ID=1",\r
 215                                                                  connection))\r
 216                     {\r
 217                         updateVersionCmd.Parameters.AddWithValue(":version", actualVersion.ToString());\r
 218                         var result = updateVersionCmd.ExecuteNonQuery();\r
 219                         Debug.Assert(result > 0);\r
 220                     }\r
 221             }\r
 222         }\r
 223 \r
 224         private void CreateTrigger()\r
 225         {\r
 226             using (var connection = GetConnection())\r
 227             using (var triggerCommand = connection.CreateCommand())\r
 228             {\r
 229                 var cmdText = new StringBuilder()\r
 230                     .AppendLine("CREATE TRIGGER IF NOT EXISTS update_last_modified UPDATE ON FileState FOR EACH ROW")\r
 231                     .AppendLine("BEGIN")\r
 232                     .AppendLine("UPDATE FileState SET Modified=datetime('now')  WHERE Id=old.Id;")\r
 233                     .AppendLine("END;")\r
 234                     .AppendLine("CREATE TRIGGER IF NOT EXISTS insert_last_modified INSERT ON FileState FOR EACH ROW")\r
 235                     .AppendLine("BEGIN")\r
 236                     .AppendLine("UPDATE FileState SET Modified=datetime('now')  WHERE Id=new.Id;")\r
 237                     .AppendLine("END;")\r
 238                     .ToString();\r
 239                 triggerCommand.CommandText = cmdText;                \r
 240                 triggerCommand.ExecuteNonQuery();\r
 241             }\r
 242         }\r
 243 \r
 244 \r
 245         private static InPlaceConfigurationSource GetConfiguration(string pithosDbPath)\r
 246         {\r
 247             if (String.IsNullOrWhiteSpace(pithosDbPath))\r
 248                 throw new ArgumentNullException("pithosDbPath");\r
 249             if (!Path.IsPathRooted(pithosDbPath))\r
 250                 throw new ArgumentException("path must be a rooted path", "pithosDbPath");\r
 251             Contract.EndContractBlock();\r
 252 \r
 253             var properties = new Dictionary<string, string>\r
 254                                  {\r
 255                                      {"connection.driver_class", "NHibernate.Driver.SQLite20Driver"},\r
 256                                      {"dialect", "NHibernate.Dialect.SQLiteDialect"},\r
 257                                      {"connection.provider", "NHibernate.Connection.DriverConnectionProvider"},\r
 258                                      {\r
 259                                          "proxyfactory.factory_class",\r
 260                                          "NHibernate.ByteCode.Castle.ProxyFactoryFactory, NHibernate.ByteCode.Castle"\r
 261                                          },\r
 262                                  };\r
 263 \r
 264             var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N", pithosDbPath);\r
 265             properties.Add("connection.connection_string", connectionString);\r
 266 \r
 267             var source = new InPlaceConfigurationSource();                        \r
 268             source.Add(typeof (ActiveRecordBase), properties);\r
 269             source.SetDebugFlag(false);            \r
 270             return source;\r
 271         }\r
 272 \r
 273         public void StartProcessing(CancellationToken token)\r
 274         {\r
 275             _persistenceAgent = Agent<Action>.Start(queue =>\r
 276             {\r
 277                 Action loop = null;\r
 278                 loop = () =>\r
 279                 {\r
 280                     var job = queue.Receive();\r
 281                     job.ContinueWith(t =>\r
 282                     {\r
 283                         var action = job.Result;\r
 284                         try\r
 285                         {\r
 286                             action();\r
 287                         }\r
 288                         catch (SQLiteException ex)\r
 289                         {\r
 290                             Log.ErrorFormat("[ERROR] SQL \n{0}", ex);\r
 291                         }\r
 292                         catch (Exception ex)\r
 293                         {\r
 294                             Log.ErrorFormat("[ERROR] STATE \n{0}", ex);\r
 295                         }\r
 296                         queue.NotifyComplete(action);\r
 297 // ReSharper disable AccessToModifiedClosure\r
 298                         queue.DoAsync(loop);\r
 299 // ReSharper restore AccessToModifiedClosure\r
 300                     });\r
 301                 };\r
 302                 loop();\r
 303             });\r
 304             \r
 305         }\r
 306 \r
 307        \r
 308 \r
 309         public void Stop()\r
 310         {\r
 311             _persistenceAgent.Stop();            \r
 312         }\r
 313 \r
 314 \r
 315         public void ProcessExistingFiles(IEnumerable<FileInfo> existingFiles)\r
 316         {\r
 317             if (existingFiles == null)\r
 318                 throw new ArgumentNullException("existingFiles");\r
 319             Contract.EndContractBlock();\r
 320 \r
 321             //Find new or matching files with a left join to the stored states\r
 322             var fileStates = FileState.Queryable.ToList();\r
 323             var currentFiles = from file in existingFiles\r
 324                                join state in fileStates on file.FullName.ToLower() equals state.FilePath.ToLower() into\r
 325                                    gs\r
 326                                from substate in gs.DefaultIfEmpty()\r
 327                                select Tuple.Create(file, substate);\r
 328 \r
 329             //To get the deleted files we must get the states that have no corresponding\r
 330             //files. \r
 331             //We can't use the File.Exists method inside a query, so we get all file paths from the states\r
 332             var statePaths = (from state in fileStates\r
 333                               select new {state.Id, state.FilePath}).ToList();\r
 334             //and check each one\r
 335             var missingStates = (from path in statePaths\r
 336                                  where !File.Exists(path.FilePath) && !Directory.Exists(path.FilePath)\r
 337                                  select path.Id).ToList();\r
 338             //Finally, retrieve the states that correspond to the deleted files            \r
 339             var deletedFiles = from state in fileStates\r
 340                                where missingStates.Contains(state.Id)\r
 341                                select Tuple.Create(default(FileInfo), state);\r
 342 \r
 343             var pairs = currentFiles.Union(deletedFiles).ToList();\r
 344 \r
 345             i = 1;\r
 346             var total = pairs.Count;\r
 347             foreach (var pair in pairs)\r
 348             {\r
 349                 ProcessFile(total, pair);\r
 350             }\r
 351         }\r
 352 \r
 353         int i = 1;\r
 354 \r
 355         private void ProcessFile(int total, Tuple<FileInfo,FileState> pair)\r
 356         {\r
 357             var idx = Interlocked.Increment(ref i);\r
 358             using (StatusNotification.GetNotifier("Indexing file {0} of {1}", "Indexed file {0} of {1} ", idx, total))\r
 359             {\r
 360                 var fileState = pair.Item2;\r
 361                 var file = pair.Item1;\r
 362                 if (fileState == null)\r
 363                 {\r
 364                     //This is a new file                        \r
 365                     var createState = FileState.CreateFor(file,StatusNotification);\r
 366                     _persistenceAgent.Post(createState.Create);\r
 367                 }\r
 368                 else if (file == null)\r
 369                 {\r
 370                     //This file was deleted while we were down. We should mark it as deleted\r
 371                     //We have to go through UpdateStatus here because the state object we are using\r
 372                     //was created by a different ORM session.\r
 373                     _persistenceAgent.Post(() => UpdateStatusDirect((Guid) fileState.Id, FileStatus.Deleted));\r
 374                 }\r
 375                 else\r
 376                 {\r
 377                     //This file has a matching state. Need to check for possible changes\r
 378                     //To check for changes, we use the cheap (in CPU terms) MD5 algorithm\r
 379                     //on the entire file.\r
 380 \r
 381                     var hashString = file.ComputeShortHash(StatusNotification);\r
 382                     Debug.Assert(hashString.Length==32);\r
 383 \r
 384 \r
 385                     //TODO: Need a way to attach the hashes to the filestate so we don't\r
 386                     //recalculate them each time a call to calculate has is made\r
 387                     //We can either store them to the filestate or add them to a \r
 388                     //dictionary\r
 389 \r
 390                     //If the hashes don't match the file was changed\r
 391                     if (fileState.ETag != hashString)\r
 392                     {\r
 393                         _persistenceAgent.Post(() => UpdateStatusDirect((Guid) fileState.Id, FileStatus.Modified));\r
 394                     }\r
 395                 }\r
 396             }\r
 397         }\r
 398 \r
 399 \r
 400         private int UpdateStatusDirect(Guid id, FileStatus status)\r
 401         {\r
 402             using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))\r
 403             {\r
 404 \r
 405                 try\r
 406                 {\r
 407                     using (var session = ActiveRecordMediator.GetSessionFactoryHolder().CreateSession(typeof(FileState)))\r
 408                     {\r
 409                         var walquery = session.CreateSQLQuery("PRAGMA journal_mode=WAL");\r
 410                         walquery.List();\r
 411 \r
 412                           //var updatecmd = session.CreateSQLQuery(\r
 413                         var updatecmd = session.CreateQuery(\r
 414                             "update FileState set FileStatus= :fileStatus where Id = :id  ")\r
 415                             .SetGuid("id", id)\r
 416                             .SetEnum("fileStatus", status);\r
 417                         var affected = updatecmd.ExecuteUpdate();\r
 418                         \r
 419                         return affected;\r
 420                     }\r
 421 \r
 422                 }\r
 423                 catch (Exception exc)\r
 424                 {\r
 425                     Log.Error(exc.ToString());\r
 426                     throw;\r
 427                 }\r
 428             }\r
 429         }\r
 430         \r
 431         private int UpdateStatusDirect(string path, FileStatus status)\r
 432         {\r
 433             using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))\r
 434             {\r
 435 \r
 436                 try\r
 437                 {                    \r
 438                     using (var session = ActiveRecordMediator.GetSessionFactoryHolder().CreateSession(typeof(FileState)))\r
 439                     using (var tx=session.BeginTransaction(IsolationLevel.ReadCommitted))\r
 440                     {\r
 441 \r
 442                         //var walquery = session.CreateSQLQuery("PRAGMA journal_mode=WAL");\r
 443                         var walquery = session.CreateQuery("PRAGMA journal_mode=WAL");\r
 444                         walquery.List();\r
 445 \r
 446                         //var updatecmd = session.CreateSQLQuery(\r
 447                         var updatecmd = session.CreateQuery(\r
 448                             "update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE")\r
 449                             .SetString("path", path)\r
 450                             .SetEnum("fileStatus", status);\r
 451                         var affected = updatecmd.ExecuteUpdate();\r
 452 \r
 453                         if (affected == 0)\r
 454                         {                            \r
 455                             var createdState = FileState.CreateFor(FileInfoExtensions.FromPath(path), StatusNotification);\r
 456                             createdState.FileStatus = status;\r
 457                             session.Save(createdState);\r
 458                         }\r
 459                         tx.Commit();\r
 460                         return affected;\r
 461                     }\r
 462                 }\r
 463                 catch (Exception exc)\r
 464                 {\r
 465                     Log.Error(exc.ToString());\r
 466                     throw;\r
 467                 }\r
 468             }\r
 469         }\r
 470 \r
 471         private int UpdateStatusDirect(string absolutePath, FileStatus fileStatus, FileOverlayStatus overlayStatus, string conflictReason)\r
 472         {\r
 473             using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))\r
 474             {\r
 475 \r
 476                 try\r
 477                 {\r
 478 \r
 479                     using (var session = ActiveRecordMediator.GetSessionFactoryHolder().CreateSession(typeof(FileState)))\r
 480                     using (var tx=session.BeginTransaction(IsolationLevel.ReadCommitted))\r
 481                     {\r
 482 \r
 483                         //var walquery = session.CreateSQLQuery("PRAGMA journal_mode=WAL");\r
 484                         var walquery = session.CreateSQLQuery("PRAGMA journal_mode=WAL");\r
 485                         walquery.List();\r
 486 \r
 487 \r
 488                         //var updatecmd = session.CreateSQLQuery("update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus,ConflictReason= :conflictReason where FilePath = :path COLLATE NOCASE")\r
 489                         var updatecmd = session.CreateQuery("update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus,ConflictReason= :conflictReason where FilePath = :path")\r
 490                                                 .SetString("path", absolutePath)\r
 491                                                 .SetEnum("fileStatus", fileStatus)\r
 492                                                 .SetEnum("overlayStatus", overlayStatus)\r
 493                                                 .SetString("conflictReason", conflictReason);\r
 494                         var affected = updatecmd.ExecuteUpdate();\r
 495 \r
 496                         if (affected == 0)\r
 497                         {\r
 498                             var createdState = FileState.CreateFor(FileInfoExtensions.FromPath(absolutePath), StatusNotification);\r
 499                             createdState.FileStatus = fileStatus;\r
 500                             createdState.OverlayStatus = overlayStatus;\r
 501                             createdState.ConflictReason = conflictReason;\r
 502                             createdState.LastMD5 = String.Empty;\r
 503                             session.Save(createdState);\r
 504                             //createdState.Create();\r
 505                         }\r
 506                         tx.Commit();\r
 507                         return affected;\r
 508                     }\r
 509                 }\r
 510                 catch (Exception exc)\r
 511                 {\r
 512                     Log.Error(exc.ToString());\r
 513                     throw;\r
 514                 }\r
 515             }\r
 516         }\r
 517         \r
 518 \r
 519 \r
 520         public string BlockHash { get; set; }\r
 521 \r
 522         public int BlockSize { get; set; }\r
 523         public void ChangeRoots(string oldPath, string newPath)\r
 524         {\r
 525             if (String.IsNullOrWhiteSpace(oldPath))\r
 526                 throw new ArgumentNullException("oldPath");\r
 527             if (!Path.IsPathRooted(oldPath))\r
 528                 throw new ArgumentException("oldPath must be an absolute path", "oldPath");\r
 529             if (string.IsNullOrWhiteSpace(newPath))\r
 530                 throw new ArgumentNullException("newPath");\r
 531             if (!Path.IsPathRooted(newPath))\r
 532                 throw new ArgumentException("newPath must be an absolute path", "newPath");\r
 533             Contract.EndContractBlock();\r
 534 \r
 535             FileState.ChangeRootPath(oldPath,newPath);\r
 536 \r
 537         }\r
 538 \r
 539 \r
 540 \r
 541         private readonly string _pithosDataPath;\r
 542 \r
 543 \r
 544         public FileState GetStateByFilePath(string path)\r
 545         {\r
 546             if (String.IsNullOrWhiteSpace(path))\r
 547                 throw new ArgumentNullException("path");\r
 548             if (!Path.IsPathRooted(path))\r
 549                 throw new ArgumentException("The path must be rooted", "path");\r
 550             Contract.EndContractBlock();\r
 551 \r
 552             try\r
 553             {\r
 554                 \r
 555                 using (var connection = GetConnection())\r
 556                 using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,ETag,Version    ,VersionTimeStamp,IsShared   ,SharedBy   ,ShareWrite, LastMD5,LastLength,LastWriteDate  from FileState where FilePath=:path COLLATE NOCASE", connection))\r
 557                 {\r
 558                     \r
 559                     command.Parameters.AddWithValue("path", path);\r
 560                     \r
 561                     using (var reader = command.ExecuteReader())\r
 562                     {\r
 563                         if (reader.Read())\r
 564                         {\r
 565                             //var values = new object[reader.FieldCount];\r
 566                             //reader.GetValues(values);\r
 567                             var state = new FileState\r
 568                                             {\r
 569                                                 Id = reader.GetGuid(0),\r
 570                                                 FilePath = reader.IsDBNull(1)?"":reader.GetString(1),\r
 571                                                 OverlayStatus =reader.IsDBNull(2)?FileOverlayStatus.Unversioned: (FileOverlayStatus) reader.GetInt64(2),\r
 572                                                 FileStatus = reader.IsDBNull(3)?FileStatus.Missing:(FileStatus) reader.GetInt64(3),\r
 573                                                 Checksum = reader.IsDBNull(4)?"":reader.GetString(4),\r
 574                                                 ETag= reader.IsDBNull(5)?"":reader.GetString(5),\r
 575                                                 Version = reader.IsDBNull(6)?default(long):reader.GetInt64(6),\r
 576                                                 VersionTimeStamp = reader.IsDBNull(7)?default(DateTime):reader.GetDateTime(7),\r
 577                                                 IsShared = !reader.IsDBNull(8) && reader.GetBoolean(8),\r
 578                                                 SharedBy = reader.IsDBNull(9)?"":reader.GetString(9),\r
 579                                                 ShareWrite = !reader.IsDBNull(10) && reader.GetBoolean(10),\r
 580                                                 LastMD5=reader.GetString(11),\r
 581                                                 LastLength=reader.IsDBNull(12)? default(long):reader.GetInt64(12),\r
 582                                                 LastWriteDate=reader.IsDBNull(13)?default(DateTime):reader.GetDateTime(13)\r
 583                                             };\r
 584 \r
 585                             return state;\r
 586                         }\r
 587                         else\r
 588                         {\r
 589                             return null;\r
 590                         }\r
 591 \r
 592                     }                    \r
 593                 }\r
 594             }\r
 595             catch (Exception exc)\r
 596             {\r
 597                 Log.ErrorFormat(exc.ToString());\r
 598                 throw;\r
 599             }            \r
 600         }\r
 601 \r
 602         public FileOverlayStatus GetFileOverlayStatus(string path)\r
 603         {\r
 604             if (String.IsNullOrWhiteSpace(path))\r
 605                 throw new ArgumentNullException("path");\r
 606             if (!Path.IsPathRooted(path))\r
 607                 throw new ArgumentException("The path must be rooted", "path");\r
 608             Contract.EndContractBlock();\r
 609 \r
 610             try\r
 611             {\r
 612                 \r
 613                 using (var connection = GetConnection())\r
 614                 using (var command = new SQLiteCommand("select OverlayStatus from FileState where FilePath=:path  COLLATE NOCASE", connection))\r
 615                 {\r
 616                     \r
 617                     command.Parameters.AddWithValue("path", path);\r
 618                     \r
 619                     var s = command.ExecuteScalar();\r
 620                     return (FileOverlayStatus) Convert.ToInt32(s);\r
 621                 }\r
 622             }\r
 623             catch (Exception exc)\r
 624             {\r
 625                 Log.ErrorFormat(exc.ToString());\r
 626                 return FileOverlayStatus.Unversioned;\r
 627             }\r
 628         }\r
 629 \r
 630         private string GetConnectionString()\r
 631         {\r
 632             var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N;Pooling=True", _pithosDataPath);\r
 633             return connectionString;\r
 634         }\r
 635 \r
 636         private SQLiteConnection GetConnection()\r
 637         {\r
 638             var connectionString = GetConnectionString();\r
 639             var connection = new SQLiteConnection(connectionString);\r
 640             connection.Open();\r
 641             using(var cmd =connection.CreateCommand())\r
 642             {\r
 643                 cmd.CommandText = "PRAGMA journal_mode=WAL";\r
 644                 cmd.ExecuteNonQuery();\r
 645             }\r
 646             return connection;\r
 647         }\r
 648 \r
 649        /* public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)\r
 650         {\r
 651             if (String.IsNullOrWhiteSpace(path))\r
 652                 throw new ArgumentNullException("path");\r
 653             if (!Path.IsPathRooted(path))\r
 654                 throw new ArgumentException("The path must be rooted","path");\r
 655             Contract.EndContractBlock();\r
 656 \r
 657             _persistenceAgent.Post(() => FileState.StoreOverlayStatus(path,overlayStatus));\r
 658         }*/\r
 659 \r
 660         public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string etag = null)\r
 661         {\r
 662             if (String.IsNullOrWhiteSpace(path))\r
 663                 throw new ArgumentNullException("path");\r
 664             if (!Path.IsPathRooted(path))\r
 665                 throw new ArgumentException("The path must be rooted","path");\r
 666             Contract.EndContractBlock();\r
 667 \r
 668             return _persistenceAgent.PostAndAwait(() => FileState.StoreOverlayStatus(path,overlayStatus,etag));\r
 669         }\r
 670 \r
 671        /* public void RenameFileOverlayStatus(string oldPath, string newPath)\r
 672         {\r
 673             if (String.IsNullOrWhiteSpace(oldPath))\r
 674                 throw new ArgumentNullException("oldPath");\r
 675             if (!Path.IsPathRooted(oldPath))\r
 676                 throw new ArgumentException("The oldPath must be rooted", "oldPath");\r
 677             if (String.IsNullOrWhiteSpace(newPath))\r
 678                 throw new ArgumentNullException("newPath");\r
 679             if (!Path.IsPathRooted(newPath))\r
 680                 throw new ArgumentException("The newPath must be rooted", "newPath");\r
 681             Contract.EndContractBlock();\r
 682 \r
 683             _persistenceAgent.Post(() =>FileState.RenameState(oldPath, newPath));\r
 684         }*/\r
 685 \r
 686         public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus, string conflictReason)\r
 687         {\r
 688             if (String.IsNullOrWhiteSpace(path))\r
 689                 throw new ArgumentNullException("path");\r
 690             if (!Path.IsPathRooted(path))\r
 691                 throw new ArgumentException("The path must be rooted", "path");\r
 692             Contract.EndContractBlock();\r
 693 \r
 694             Debug.Assert(!path.Contains(FolderConstants.CacheFolder));\r
 695             Debug.Assert(!path.EndsWith(".ignore"));\r
 696 \r
 697             _persistenceAgent.Post(() => UpdateStatusDirect(path, fileStatus, overlayStatus, conflictReason));\r
 698         }\r
 699 \r
 700         \r
 701         public void StoreInfo(string path, ObjectInfo objectInfo)\r
 702         {\r
 703             if (String.IsNullOrWhiteSpace(path))\r
 704                 throw new ArgumentNullException("path");\r
 705             if (!Path.IsPathRooted(path))\r
 706                 throw new ArgumentException("The path must be rooted", "path");\r
 707             if (objectInfo == null)\r
 708                 throw new ArgumentNullException("objectInfo", "objectInfo can't be empty");\r
 709             Contract.EndContractBlock();\r
 710 \r
 711             _persistenceAgent.Post(() => StoreInfoDirect(path, objectInfo));\r
 712 \r
 713         }\r
 714 \r
 715         private void StoreInfoDirect(string path, ObjectInfo objectInfo)\r
 716         {\r
 717             try\r
 718             {\r
 719                     using (var session = ActiveRecordMediator.GetSessionFactoryHolder().CreateSession(typeof(FileState)))\r
 720                     using (var tx=session.BeginTransaction(IsolationLevel.ReadCommitted))\r
 721                     {\r
 722 \r
 723                         //var walquery = session.CreateSQLQuery("PRAGMA journal_mode=WAL");\r
 724                         var walquery = session.CreateSQLQuery("PRAGMA journal_mode=WAL");\r
 725                         walquery.List();\r
 726 \r
 727                         //An entry for the new path may exist, \r
 728                         IQuery deletecmd = session.CreateQuery(\r
 729                            "delete from FileState where FilePath=:path and ObjectID is null")\r
 730                            .SetString("path",path);                        \r
 731                         deletecmd.ExecuteUpdate();\r
 732 \r
 733 \r
 734                         Func<IQuery, IQuery> setCriteria = q => q\r
 735                                                     .SetString("path", path)\r
 736                                                     .SetString("checksum",objectInfo.X_Object_Hash)\r
 737                                                     .SetString("etag", objectInfo.ETag)\r
 738                                                     .SetInt64("version", objectInfo.Version.GetValueOrDefault())\r
 739                                                     .SetDateTime("versionTimeStamp",objectInfo.VersionTimestamp.GetValueOrDefault())\r
 740                                                     .SetEnum("fileStatus", FileStatus.Unchanged)\r
 741                                                     .SetEnum("overlayStatus",FileOverlayStatus.Normal)\r
 742                                                     .SetString("objectID", objectInfo.UUID);\r
 743                         //IQuery updatecmd = session.CreateSQLQuery(\r
 744                         IQuery updatecmd = session.CreateQuery(\r
 745                             "update FileState set FilePath=:path,FileStatus= :fileStatus,OverlayStatus= :overlayStatus, Checksum=:checksum, ETag=:etag,LastMD5=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp where ObjectID = :objectID  ");\r
 746                         updatecmd = setCriteria(updatecmd);                           \r
 747                         var affected = updatecmd.ExecuteUpdate();                        \r
 748                         \r
 749                     //If the ID exists, update the status\r
 750                     if (affected == 0)\r
 751                     {\r
 752                         //If the ID doesn't exist, try to update using the path, and store the ID as well.\r
 753                         //updatecmd = session.CreateSQLQuery(\r
 754                         updatecmd = session.CreateQuery(\r
 755                         //    "update FileState set FileStatus= :fileStatus,OverlayStatus= :overlayStatus, ObjectID=:objectID, Checksum=:checksum, ETag=:etag,LastMD5=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp where FilePath = :path  COLLATE NOCASE ");\r
 756                             "update FileState set FileStatus= :fileStatus,OverlayStatus= :overlayStatus, ObjectID=:objectID, Checksum=:checksum, ETag=:etag,LastMD5=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp where FilePath = :path");\r
 757                         updatecmd=setCriteria(updatecmd);\r
 758                         affected = updatecmd.ExecuteUpdate();\r
 759                     }\r
 760                     if (affected==0)\r
 761                     {\r
 762                         //IQuery insertCmd=session.CreateSQLQuery(\r
 763                         IQuery insertCmd = session.CreateSQLQuery(\r
 764                             "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,ETag,LastMD5,FileStatus,OverlayStatus,ObjectID) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:etag,:etag,:fileStatus,:overlayStatus,:objectID)");\r
 765                         insertCmd=setCriteria(insertCmd).SetGuid("id", Guid.NewGuid());\r
 766                         affected = insertCmd.ExecuteUpdate();\r
 767                     }\r
 768                     tx.Commit();\r
 769                 }\r
 770             }\r
 771             catch (Exception exc)\r
 772             {\r
 773                 Log.ErrorFormat("Failed to update [{0}]:[{1}]\r\n{2}",path,objectInfo.UUID, exc);\r
 774                 throw;\r
 775             }\r
 776         }\r
 777 \r
 778         private bool StateExists(string filePath,SQLiteConnection connection)\r
 779         {\r
 780             using (var command = new SQLiteCommand("Select count(*) from FileState where FilePath=:path  COLLATE NOCASE", connection))\r
 781             {\r
 782                 command.Parameters.AddWithValue("path", filePath);\r
 783                 var result = command.ExecuteScalar();\r
 784                 return ((long)result >= 1);\r
 785             }\r
 786 \r
 787         }\r
 788 \r
 789         private bool StateExistsByID(string objectId,SQLiteConnection connection)\r
 790         {\r
 791             using (var command = new SQLiteCommand("Select count(*) from FileState where ObjectId=:id", connection))\r
 792             {\r
 793                 command.Parameters.AddWithValue("id", objectId);\r
 794                 var result = command.ExecuteScalar();\r
 795                 return ((long)result >= 1);\r
 796             }\r
 797 \r
 798         }\r
 799 \r
 800         public void SetFileStatus(string path, FileStatus status)\r
 801         {\r
 802             if (String.IsNullOrWhiteSpace(path))\r
 803                 throw new ArgumentNullException("path");\r
 804             if (!Path.IsPathRooted(path))\r
 805                 throw new ArgumentException("The path must be rooted", "path");\r
 806             Contract.EndContractBlock();\r
 807             \r
 808             _persistenceAgent.Post(() => UpdateStatusDirect(path, status));\r
 809         }\r
 810 \r
 811         public FileStatus GetFileStatus(string path)\r
 812         {\r
 813             if (String.IsNullOrWhiteSpace(path))\r
 814                 throw new ArgumentNullException("path");\r
 815             if (!Path.IsPathRooted(path))\r
 816                 throw new ArgumentException("The path must be rooted", "path");\r
 817             Contract.EndContractBlock();\r
 818 \r
 819             \r
 820             using (var connection = GetConnection())\r
 821             {\r
 822                 var command = new SQLiteCommand("select FileStatus from FileState where FilePath=:path  COLLATE NOCASE", connection);\r
 823                 command.Parameters.AddWithValue("path", path);\r
 824                 \r
 825                 var statusValue = command.ExecuteScalar();\r
 826                 if (statusValue==null)\r
 827                     return FileStatus.Missing;\r
 828                 return (FileStatus)Convert.ToInt32(statusValue);\r
 829             }\r
 830         }\r
 831 \r
 832         /// <summary>\r
 833         /// Deletes the status of the specified file\r
 834         /// </summary>\r
 835         /// <param name="path"></param>\r
 836         public void ClearFileStatus(string path)\r
 837         {\r
 838             if (String.IsNullOrWhiteSpace(path))\r
 839                 throw new ArgumentNullException("path");\r
 840             if (!Path.IsPathRooted(path))\r
 841                 throw new ArgumentException("The path must be rooted", "path");\r
 842             Contract.EndContractBlock();\r
 843 \r
 844             _persistenceAgent.Post(() => DeleteDirect(path));   \r
 845         }\r
 846 \r
 847         /// <summary>\r
 848         /// Deletes the status of the specified folder and all its contents\r
 849         /// </summary>\r
 850         /// <param name="path"></param>\r
 851         public void ClearFolderStatus(string path)\r
 852         {\r
 853             if (String.IsNullOrWhiteSpace(path))\r
 854                 throw new ArgumentNullException("path");\r
 855             if (!Path.IsPathRooted(path))\r
 856                 throw new ArgumentException("The path must be rooted", "path");\r
 857             Contract.EndContractBlock();\r
 858             //The agent may be null when removing an invalid/expired account from Settings, in which case processing hasn't yet started.\r
 859             if (_persistenceAgent == null)\r
 860                 //In this case remove the folder immediatelly\r
 861                 DeleteFolderDirect(path);\r
 862             else\r
 863                 //Otherwise schedule a delete\r
 864                 _persistenceAgent.Post(() => DeleteFolderDirect(path));   \r
 865         }\r
 866 \r
 867         public IEnumerable<FileState> GetChildren(FileState fileState)\r
 868         {\r
 869             if (fileState == null)\r
 870                 throw new ArgumentNullException("fileState");\r
 871             Contract.EndContractBlock();\r
 872 \r
 873             var children = from state in FileState.Queryable\r
 874                            where state.FilePath.StartsWith(fileState.FilePath + "\\")\r
 875                            select state;\r
 876             return children;\r
 877         }\r
 878 \r
 879         public void EnsureFileState(string path)\r
 880         {\r
 881             var existingState = GetStateByFilePath(path);\r
 882             if (existingState != null)\r
 883                 return;\r
 884             var fileInfo = FileInfoExtensions.FromPath(path);\r
 885             using (new SessionScope())\r
 886             {\r
 887                 var newState = FileState.CreateFor(fileInfo,StatusNotification);\r
 888                 newState.FileStatus=FileStatus.Missing;\r
 889                 _persistenceAgent.PostAndAwait(newState.CreateAndFlush).Wait();\r
 890             }\r
 891 \r
 892         }\r
 893 \r
 894         private int DeleteDirect(string filePath)\r
 895         {\r
 896             using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))\r
 897             {\r
 898 \r
 899                 try\r
 900                 {\r
 901 \r
 902                     \r
 903                     using (var connection = GetConnection())\r
 904                     {\r
 905                         var command = new SQLiteCommand("delete from FileState where FilePath = :path  COLLATE NOCASE",\r
 906                                                         connection);\r
 907 \r
 908                         command.Parameters.AddWithValue("path", filePath);\r
 909                         \r
 910                         var affected = command.ExecuteNonQuery();\r
 911                         return affected;\r
 912                     }\r
 913                 }\r
 914                 catch (Exception exc)\r
 915                 {\r
 916                     Log.Error(exc.ToString());\r
 917                     throw;\r
 918                 }\r
 919             }\r
 920         }\r
 921 \r
 922         private int DeleteFolderDirect(string filePath)\r
 923         {\r
 924             using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))\r
 925             {\r
 926 \r
 927                 try\r
 928                 {\r
 929 \r
 930                     \r
 931                     using (var connection = GetConnection())\r
 932                     {\r
 933                         var command = new SQLiteCommand(@"delete from FileState where FilePath = :path or FilePath like :path || '\%'  COLLATE NOCASE",\r
 934                                                         connection);\r
 935 \r
 936                         command.Parameters.AddWithValue("path", filePath);\r
 937                         \r
 938                         var affected = command.ExecuteNonQuery();\r
 939                         return affected;\r
 940                     }\r
 941                 }\r
 942                 catch (Exception exc)\r
 943                 {\r
 944                     Log.Error(exc.ToString());\r
 945                     throw;\r
 946                 }\r
 947             }\r
 948         }\r
 949 \r
 950 \r
 951         \r
 952         public void UpdateFileTreeHash(string path, TreeHash treeHash)\r
 953         {\r
 954             if (String.IsNullOrWhiteSpace(path))\r
 955                 throw new ArgumentNullException("path");\r
 956             if (!Path.IsPathRooted(path))\r
 957                 throw new ArgumentException("The path must be rooted", "path");            \r
 958             if (treeHash==null)\r
 959                 throw new ArgumentNullException("treeHash");\r
 960             Contract.EndContractBlock();\r
 961 \r
 962             _persistenceAgent.Post(() => FileState.UpdateFileTreeHash(path, treeHash));\r
 963         }\r
 964 \r
 965         public void UpdateFileChecksum(string path, string etag, string checksum)\r
 966         {\r
 967             if (String.IsNullOrWhiteSpace(path))\r
 968                 throw new ArgumentNullException("path");\r
 969             if (!Path.IsPathRooted(path))\r
 970                 throw new ArgumentException("The path must be rooted", "path");            \r
 971             Contract.EndContractBlock();\r
 972 \r
 973             _persistenceAgent.Post(() => FileState.UpdateChecksum(path, etag,checksum));\r
 974         }\r
 975 \r
 976         public void UpdateLastMD5(FileInfo file, string etag)\r
 977         {\r
 978             if (file==null)\r
 979                 throw new ArgumentNullException("file");\r
 980             if (String.IsNullOrWhiteSpace(etag))\r
 981                 throw new ArgumentNullException("etag");\r
 982             Contract.EndContractBlock();\r
 983 \r
 984             _persistenceAgent.Post(() => FileState.UpdateLastMD5(file, etag));\r
 985         }\r
 986 \r
 987 \r
 988         public void CleanupOrphanStates()\r
 989         {\r
 990             //Orphan states are those that do not correspond to an account, ie. their paths\r
 991             //do not start with the root path of any registered account\r
 992 \r
 993             var roots=(from account in Settings.Accounts\r
 994                       select account.RootPath).ToList();\r
 995             \r
 996             var allStates = from state in FileState.Queryable\r
 997                 select state.FilePath;\r
 998 \r
 999             foreach (var statePath in allStates)\r
1000             {\r
1001                 if (!roots.Any(root=>statePath.StartsWith(root,StringComparison.InvariantCultureIgnoreCase)))\r
1002                     this.DeleteDirect(statePath);\r
1003             }\r
1004         }\r
1005 \r
1006         public void CleanupStaleStates(AccountInfo accountInfo, List<ObjectInfo> objectInfos)\r
1007         {\r
1008             if (accountInfo == null)\r
1009                 throw new ArgumentNullException("accountInfo");\r
1010             if (objectInfos == null)\r
1011                 throw new ArgumentNullException("objectInfos");\r
1012             Contract.EndContractBlock();\r
1013             \r
1014 \r
1015 \r
1016             //Stale states are those that have no corresponding local or server file\r
1017             \r
1018 \r
1019             var agent=FileAgent.GetFileAgent(accountInfo);\r
1020 \r
1021             var localFiles=agent.EnumerateFiles();\r
1022             var localSet = new HashSet<string>(localFiles);\r
1023 \r
1024             //RelativeUrlToFilePath will fail for\r
1025             //infos of accounts, containers which have no Name\r
1026 \r
1027             var serverFiles = from info in objectInfos\r
1028                               where info.Name != null\r
1029                               select Path.Combine(accountInfo.AccountPath,info.RelativeUrlToFilePath(accountInfo.UserName));\r
1030             var serverSet = new HashSet<string>(serverFiles);\r
1031 \r
1032             var allStates = from state in FileState.Queryable\r
1033                             where state.FilePath.StartsWith(agent.RootPath)\r
1034                             select state.FilePath;\r
1035             var stateSet = new HashSet<string>(allStates);\r
1036             stateSet.ExceptWith(serverSet);\r
1037             stateSet.ExceptWith(localSet);\r
1038 \r
1039             foreach (var remainder in stateSet)\r
1040             {\r
1041                 DeleteDirect(remainder);\r
1042             }\r
1043 \r
1044             \r
1045         }\r
1046     }\r
1047 \r
1048    \r
1049 }\r