code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/StatusAgent.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="StatusAgent.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System;
  43 using System.Collections.Generic;
  44 using System.ComponentModel.Composition;
  45 using System.Data;
  46 using System.Data.SQLite;
  47 using System.Diagnostics;
  48 using System.Diagnostics.Contracts;
  49 using System.IO;
  50 using System.Linq;
  51 using System.Reflection;
  52 using System.Security.Cryptography;
  53 using System.Text;
  54 using System.Threading;
  55 using System.Threading.Tasks;
  56 using Castle.ActiveRecord;
  57 using Castle.ActiveRecord.Framework;
  58 using Castle.ActiveRecord.Framework.Config;
  59 using Castle.ActiveRecord.Queries;
  60 using NHibernate;
  61 using NHibernate.ByteCode.Castle;
  62 using NHibernate.Cfg;
  63 using NHibernate.Cfg.Loquacious;
  64 using NHibernate.Dialect;
  65 using NHibernate.Exceptions;
  66 using Pithos.Interfaces;
  67 using Pithos.Network;
  68 using log4net;
  69 using Environment = System.Environment;
  70
  71 namespace Pithos.Core.Agents
  72 {
  73     [Export(typeof(IStatusChecker)),Export(typeof(IStatusKeeper))]
  74     public class StatusAgent:IStatusChecker,IStatusKeeper
  75     {
  76         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  77
  78         [System.ComponentModel.Composition.Import]
  79         public IPithosSettings Settings { get; set; }
  80
  81         [System.ComponentModel.Composition.Import]
  82         public IStatusNotification StatusNotification { get; set; }
  83
  84         private Agent<Action> _persistenceAgent;
  85
  86
  87
  88         public StatusAgent()
  89         {
  90             var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
  91
  92             _pithosDataPath = Path.Combine(appDataPath , "GRNET\\PITHOS");
  93             if (!Directory.Exists(_pithosDataPath))
  94                 Directory.CreateDirectory(_pithosDataPath);
  95
  96             var dbPath = Path.Combine(_pithosDataPath, "pithos.db");
  97
  98             MigrateOldDb(dbPath, appDataPath);
  99
 100
 101             var source = GetConfiguration(_pithosDataPath);
 102             ActiveRecordStarter.Initialize(source,typeof(FileState),typeof(FileTag));
 103
 104             UpgradeDatabase();
 105
 106
 107
 108             if (!File.Exists(dbPath))
 109                 ActiveRecordStarter.CreateSchema();
 110
 111             CreateTrigger();
 112
 113         }
 114
 115
 116         private static void MigrateOldDb(string dbPath, string appDataPath)
 117         {
 118             if(String.IsNullOrWhiteSpace(dbPath))
 119                 throw new ArgumentNullException("dbPath");
 120             if(String.IsNullOrWhiteSpace(appDataPath))
 121                 throw new ArgumentNullException("appDataPath");
 122             Contract.EndContractBlock();
 123
 124             var oldDbPath = Path.Combine(appDataPath, "Pithos", "pithos.db");
 125             var oldDbInfo = new FileInfo(oldDbPath);
 126             if (oldDbInfo.Exists && !File.Exists(dbPath))
 127             {
 128                 Log.InfoFormat("Moving database from {0} to {1}",oldDbInfo.FullName,dbPath);
 129                 var oldDirectory = oldDbInfo.Directory;
 130                 oldDbInfo.MoveTo(dbPath);
 131
 132                 if (Log.IsDebugEnabled)
 133                     Log.DebugFormat("Deleting {0}",oldDirectory.FullName);
 134
 135                 oldDirectory.Delete(true);
 136             }
 137         }
 138
 139         private T? GetNull<T>(string commandText,SQLiteConnection connection) where T:struct
 140         {
 141             using (var command= new SQLiteCommand(commandText, connection))
 142             {
 143                 var result = command.ExecuteScalar();
 144                 if (result == null)
 145                     return null;
 146                 return (T)result;
 147             }
 148         }
 149
 150         private T Get<T>(string commandText,SQLiteConnection connection)
 151         {
 152             using (var command= new SQLiteCommand(commandText, connection))
 153             {
 154                 var result = command.ExecuteScalar();
 155                 if (result == null)
 156                     return default(T);
 157                 return (T)result;
 158             }
 159         }
 160
 161         private int Run(string commandText,SQLiteConnection connection)
 162         {
 163             using (var command= new SQLiteCommand(commandText, connection))
 164             {
 165                 var result=command.ExecuteNonQuery();
 166                 return result;
 167             }
 168         }
 169
 170         private void UpgradeDatabase()
 171         {
 172             const string hasVersionText = "select 1 from sqlite_master where name='Version'";
 173
 174             const string getVersionCmd = "select Version from version where Id=1";
 175
 176             const string createVersionCmd = "create table Version(Id integer,Version TEXT);\n" +
 177                                             "INSERT INTO VERSION (Id,Version) VALUES(1,'0.0.0.0');";
 178
 179             const string upgradeText = "PRAGMA writable_schema = 1;\n" +
 180                                    "UPDATE SQLITE_MASTER SET SQL = 'CREATE TABLE FileState (Id UNIQUEIDENTIFIER not null, ObjectID TEXT COLLATE NOCASE, FilePath TEXT unique COLLATE NOCASE, OverlayStatus INTEGER, FileStatus INTEGER, ConflictReason TEXT, Checksum TEXT COLLATE NOCASE, ETag TEXT not null COLLATE NOCASE, LastMD5 TEXT not null COLLATE NOCASE, LastWriteDate DATETIME, LastLength INTEGER, Version INTEGER, VersionTimeStamp DATETIME, IsShared INTEGER, SharedBy TEXT, ShareWrite INTEGER, IsFolder INTEGER, Modified DATETIME, primary key (Id),unique (FilePath))' WHERE NAME = 'FileState';\n" +
 181                                    "PRAGMA writable_schema = 0;\n" +
 182                                    "VACUUM;";
 183
 184             using (var connection = GetConnection())
 185             {
 186                 var hasVersion = false;
 187                 hasVersion = GetNull<long>(hasVersionText, connection).HasValue;
 188
 189                 var storedVersion = new Version();
 190
 191                 if (hasVersion)
 192                 {
 193                     var versionTxt = Get<string>(getVersionCmd, connection);
 194                     storedVersion = new Version(versionTxt);
 195                 }
 196                 else
 197                     Run(createVersionCmd, connection);
 198
 199                 var actualVersion = Assembly.GetEntryAssembly().GetName().Version;
 200                 if (!hasVersion || actualVersion > storedVersion)
 201                     Run(upgradeText, connection);
 202
 203                 if (actualVersion != storedVersion)
 204                     using (var updateVersionCmd = new SQLiteCommand("UPDATE VERSION SET Version=:version where ID=1",
 205                                                                  connection))
 206                     {
 207                         updateVersionCmd.Parameters.AddWithValue(":version", actualVersion.ToString());
 208                         var result = updateVersionCmd.ExecuteNonQuery();
 209                         Debug.Assert(result > 0);
 210                     }
 211             }
 212         }
 213
 214         private void CreateTrigger()
 215         {
 216             using (var connection = GetConnection())
 217             using (var triggerCommand = connection.CreateCommand())
 218             {
 219                 var cmdText = new StringBuilder()
 220                     .AppendLine("CREATE TRIGGER IF NOT EXISTS update_last_modified UPDATE ON FileState FOR EACH ROW")
 221                     .AppendLine("BEGIN")
 222                     .AppendLine("UPDATE FileState SET Modified=datetime('now')  WHERE Id=old.Id;")
 223                     .AppendLine("END;")
 224                     .AppendLine("CREATE TRIGGER IF NOT EXISTS insert_last_modified INSERT ON FileState FOR EACH ROW")
 225                     .AppendLine("BEGIN")
 226                     .AppendLine("UPDATE FileState SET Modified=datetime('now')  WHERE Id=new.Id;")
 227                     .AppendLine("END;")
 228                     .ToString();
 229                 triggerCommand.CommandText = cmdText;
 230                 triggerCommand.ExecuteNonQuery();
 231             }
 232         }
 233
 234
 235         private static InPlaceConfigurationSource GetConfiguration(string pithosDbPath)
 236         {
 237             if (String.IsNullOrWhiteSpace(pithosDbPath))
 238                 throw new ArgumentNullException("pithosDbPath");
 239             if (!Path.IsPathRooted(pithosDbPath))
 240                 throw new ArgumentException("path must be a rooted path", "pithosDbPath");
 241             Contract.EndContractBlock();
 242
 243             var properties = new Dictionary<string, string>
 244                                  {
 245                                      {"connection.driver_class", "NHibernate.Driver.SQLite20Driver"},
 246                                      {"dialect", "NHibernate.Dialect.SQLiteDialect"},
 247                                      {"connection.provider", "NHibernate.Connection.DriverConnectionProvider"},
 248                                      {
 249                                          "proxyfactory.factory_class",
 250                                          "NHibernate.ByteCode.Castle.ProxyFactoryFactory, NHibernate.ByteCode.Castle"
 251                                          },
 252                                  };
 253
 254             var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N", pithosDbPath);
 255             properties.Add("connection.connection_string", connectionString);
 256
 257             var source = new InPlaceConfigurationSource();
 258             source.Add(typeof (ActiveRecordBase), properties);
 259             source.SetDebugFlag(false);
 260             return source;
 261         }
 262
 263         public void StartProcessing(CancellationToken token)
 264         {
 265             _persistenceAgent = Agent<Action>.Start(queue =>
 266             {
 267                 Action loop = null;
 268                 loop = () =>
 269                 {
 270                     var job = queue.Receive();
 271                     job.ContinueWith(t =>
 272                     {
 273                         var action = job.Result;
 274                         try
 275                         {
 276                             action();
 277                         }
 278                         catch (SQLiteException ex)
 279                         {
 280                             Log.ErrorFormat("[ERROR] SQL \n{0}", ex);
 281                         }
 282                         catch (Exception ex)
 283                         {
 284                             Log.ErrorFormat("[ERROR] STATE \n{0}", ex);
 285                         }
 286                         queue.NotifyComplete(action);
 287 // ReSharper disable AccessToModifiedClosure
 288                         queue.DoAsync(loop);
 289 // ReSharper restore AccessToModifiedClosure
 290                     });
 291                 };
 292                 loop();
 293             });
 294
 295         }
 296
 297
 298
 299         public void Stop()
 300         {
 301             _persistenceAgent.Stop();
 302         }
 303
 304
 305         public void ProcessExistingFiles(IEnumerable<FileInfo> existingFiles)
 306         {
 307             if (existingFiles == null)
 308                 throw new ArgumentNullException("existingFiles");
 309             Contract.EndContractBlock();
 310
 311             //Find new or matching files with a left join to the stored states
 312             var fileStates = FileState.Queryable.ToList();
 313             var currentFiles = from file in existingFiles
 314                                join state in fileStates on file.FullName.ToLower() equals state.FilePath.ToLower() into
 315                                    gs
 316                                from substate in gs.DefaultIfEmpty()
 317                                select Tuple.Create(file, substate);
 318
 319             //To get the deleted files we must get the states that have no corresponding
 320             //files.
 321             //We can't use the File.Exists method inside a query, so we get all file paths from the states
 322             var statePaths = (from state in fileStates
 323                               select new {state.Id, state.FilePath}).ToList();
 324             //and check each one
 325             var missingStates = (from path in statePaths
 326                                  where !File.Exists(path.FilePath) && !Directory.Exists(path.FilePath)
 327                                  select path.Id).ToList();
 328             //Finally, retrieve the states that correspond to the deleted files
 329             var deletedFiles = from state in fileStates
 330                                where missingStates.Contains(state.Id)
 331                                select Tuple.Create(default(FileInfo), state);
 332
 333             var pairs = currentFiles.Union(deletedFiles).ToList();
 334
 335             i = 1;
 336             var total = pairs.Count;
 337             foreach (var pair in pairs)
 338             {
 339                 ProcessFile(total, pair);
 340             }
 341         }
 342
 343         int i = 1;
 344
 345         private void ProcessFile(int total, Tuple<FileInfo,FileState> pair)
 346         {
 347             var idx = Interlocked.Increment(ref i);
 348             using (StatusNotification.GetNotifier("Indexing file {0} of {1}", "Indexed file {0} of {1} ", idx, total))
 349             {
 350                 var fileState = pair.Item2;
 351                 var file = pair.Item1;
 352                 if (fileState == null)
 353                 {
 354                     //This is a new file
 355                     var createState = FileState.CreateFor(file,StatusNotification);
 356                     _persistenceAgent.Post(createState.Create);
 357                 }
 358                 else if (file == null)
 359                 {
 360                     //This file was deleted while we were down. We should mark it as deleted
 361                     //We have to go through UpdateStatus here because the state object we are using
 362                     //was created by a different ORM session.
 363                     _persistenceAgent.Post(() => UpdateStatusDirect((Guid) fileState.Id, FileStatus.Deleted));
 364                 }
 365                 else
 366                 {
 367                     //This file has a matching state. Need to check for possible changes
 368                     //To check for changes, we use the cheap (in CPU terms) MD5 algorithm
 369                     //on the entire file.
 370
 371                     var hashString = file.ComputeShortHash(StatusNotification);
 372                     Debug.Assert(hashString.Length==32);
 373
 374
 375                     //TODO: Need a way to attach the hashes to the filestate so we don't
 376                     //recalculate them each time a call to calculate has is made
 377                     //We can either store them to the filestate or add them to a
 378                     //dictionary
 379
 380                     //If the hashes don't match the file was changed
 381                     if (fileState.ETag != hashString)
 382                     {
 383                         _persistenceAgent.Post(() => UpdateStatusDirect((Guid) fileState.Id, FileStatus.Modified));
 384                     }
 385                 }
 386             }
 387         }
 388
 389
 390         private int UpdateStatusDirect(Guid id, FileStatus status)
 391         {
 392             using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
 393             {
 394
 395                 try
 396                 {
 397                     using (var session = ActiveRecordMediator.GetSessionFactoryHolder().CreateSession(typeof(FileState)))
 398                     {
 399                         var walquery = session.CreateSQLQuery("PRAGMA journal_mode=WAL");
 400                         walquery.List();
 401
 402                           //var updatecmd = session.CreateSQLQuery(
 403                         var updatecmd = session.CreateQuery(
 404                             "update FileState set FileStatus= :fileStatus where Id = :id  ")
 405                             .SetGuid("id", id)
 406                             .SetEnum("fileStatus", status);
 407                         var affected = updatecmd.ExecuteUpdate();
 408
 409                         return affected;
 410                     }
 411
 412                 }
 413                 catch (Exception exc)
 414                 {
 415                     Log.Error(exc.ToString());
 416                     throw;
 417                 }
 418             }
 419         }
 420
 421         private int UpdateStatusDirect(string path, FileStatus status)
 422         {
 423             using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
 424             {
 425
 426                 try
 427                 {
 428                     using (var session = ActiveRecordMediator.GetSessionFactoryHolder().CreateSession(typeof(FileState)))
 429                     using (var tx=session.BeginTransaction(IsolationLevel.ReadCommitted))
 430                     {
 431
 432                         //var walquery = session.CreateSQLQuery("PRAGMA journal_mode=WAL");
 433                         var walquery = session.CreateQuery("PRAGMA journal_mode=WAL");
 434                         walquery.List();
 435
 436                         //var updatecmd = session.CreateSQLQuery(
 437                         var updatecmd = session.CreateQuery(
 438                             "update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE")
 439                             .SetString("path", path)
 440                             .SetEnum("fileStatus", status);
 441                         var affected = updatecmd.ExecuteUpdate();
 442
 443                         if (affected == 0)
 444                         {
 445                             var createdState = FileState.CreateFor(FileInfoExtensions.FromPath(path), StatusNotification);
 446                             createdState.FileStatus = status;
 447                             session.Save(createdState);
 448                         }
 449                         tx.Commit();
 450                         return affected;
 451                     }
 452                 }
 453                 catch (Exception exc)
 454                 {
 455                     Log.Error(exc.ToString());
 456                     throw;
 457                 }
 458             }
 459         }
 460
 461         private int UpdateStatusDirect(string absolutePath, FileStatus fileStatus, FileOverlayStatus overlayStatus, string conflictReason)
 462         {
 463             using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
 464             {
 465
 466                 try
 467                 {
 468
 469                     using (var session = ActiveRecordMediator.GetSessionFactoryHolder().CreateSession(typeof(FileState)))
 470                     using (var tx=session.BeginTransaction(IsolationLevel.ReadCommitted))
 471                     {
 472
 473                         //var walquery = session.CreateSQLQuery("PRAGMA journal_mode=WAL");
 474                         var walquery = session.CreateSQLQuery("PRAGMA journal_mode=WAL");
 475                         walquery.List();
 476
 477
 478                         //var updatecmd = session.CreateSQLQuery("update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus,ConflictReason= :conflictReason where FilePath = :path COLLATE NOCASE")
 479                         var updatecmd = session.CreateQuery("update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus,ConflictReason= :conflictReason where FilePath = :path")
 480                                                 .SetString("path", absolutePath)
 481                                                 .SetEnum("fileStatus", fileStatus)
 482                                                 .SetEnum("overlayStatus", overlayStatus)
 483                                                 .SetString("conflictReason", conflictReason);
 484                         var affected = updatecmd.ExecuteUpdate();
 485
 486                         if (affected == 0)
 487                         {
 488                             var createdState = FileState.CreateFor(FileInfoExtensions.FromPath(absolutePath), StatusNotification);
 489                             createdState.FileStatus = fileStatus;
 490                             createdState.OverlayStatus = overlayStatus;
 491                             createdState.ConflictReason = conflictReason;
 492                             createdState.LastMD5 = String.Empty;
 493                             session.Save(createdState);
 494                             //createdState.Create();
 495                         }
 496                         tx.Commit();
 497                         return affected;
 498                     }
 499                 }
 500                 catch (Exception exc)
 501                 {
 502                     Log.Error(exc.ToString());
 503                     throw;
 504                 }
 505             }
 506         }
 507
 508
 509
 510         public string BlockHash { get; set; }
 511
 512         public int BlockSize { get; set; }
 513         public void ChangeRoots(string oldPath, string newPath)
 514         {
 515             if (String.IsNullOrWhiteSpace(oldPath))
 516                 throw new ArgumentNullException("oldPath");
 517             if (!Path.IsPathRooted(oldPath))
 518                 throw new ArgumentException("oldPath must be an absolute path", "oldPath");
 519             if (string.IsNullOrWhiteSpace(newPath))
 520                 throw new ArgumentNullException("newPath");
 521             if (!Path.IsPathRooted(newPath))
 522                 throw new ArgumentException("newPath must be an absolute path", "newPath");
 523             Contract.EndContractBlock();
 524
 525             FileState.ChangeRootPath(oldPath,newPath);
 526
 527         }
 528
 529
 530
 531         private readonly string _pithosDataPath;
 532
 533
 534         public FileState GetStateByFilePath(string path)
 535         {
 536             if (String.IsNullOrWhiteSpace(path))
 537                 throw new ArgumentNullException("path");
 538             if (!Path.IsPathRooted(path))
 539                 throw new ArgumentException("The path must be rooted", "path");
 540             Contract.EndContractBlock();
 541
 542             try
 543             {
 544
 545                 using (var connection = GetConnection())
 546                 using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,ETag,Version    ,VersionTimeStamp,IsShared   ,SharedBy   ,ShareWrite, LastMD5,LastLength,LastWriteDate  from FileState where FilePath=:path COLLATE NOCASE", connection))
 547                 {
 548
 549                     command.Parameters.AddWithValue("path", path);
 550
 551                     using (var reader = command.ExecuteReader())
 552                     {
 553                         if (reader.Read())
 554                         {
 555                             //var values = new object[reader.FieldCount];
 556                             //reader.GetValues(values);
 557                             var state = new FileState
 558                                             {
 559                                                 Id = reader.GetGuid(0),
 560                                                 FilePath = reader.IsDBNull(1)?"":reader.GetString(1),
 561                                                 OverlayStatus =reader.IsDBNull(2)?FileOverlayStatus.Unversioned: (FileOverlayStatus) reader.GetInt64(2),
 562                                                 FileStatus = reader.IsDBNull(3)?FileStatus.Missing:(FileStatus) reader.GetInt64(3),
 563                                                 Checksum = reader.IsDBNull(4)?"":reader.GetString(4),
 564                                                 ETag= reader.IsDBNull(5)?"":reader.GetString(5),
 565                                                 Version = reader.IsDBNull(6)?default(long):reader.GetInt64(6),
 566                                                 VersionTimeStamp = reader.IsDBNull(7)?default(DateTime):reader.GetDateTime(7),
 567                                                 IsShared = !reader.IsDBNull(8) && reader.GetBoolean(8),
 568                                                 SharedBy = reader.IsDBNull(9)?"":reader.GetString(9),
 569                                                 ShareWrite = !reader.IsDBNull(10) && reader.GetBoolean(10),
 570                                                 LastMD5=reader.GetString(11),
 571                                                 LastLength=reader.IsDBNull(12)? default(long):reader.GetInt64(12),
 572                                                 LastWriteDate=reader.IsDBNull(13)?default(DateTime):reader.GetDateTime(13)
 573                                             };
 574
 575                             return state;
 576                         }
 577                         else
 578                         {
 579                             return null;
 580                         }
 581
 582                     }
 583                 }
 584             }
 585             catch (Exception exc)
 586             {
 587                 Log.ErrorFormat(exc.ToString());
 588                 throw;
 589             }
 590         }
 591
 592         public FileOverlayStatus GetFileOverlayStatus(string path)
 593         {
 594             if (String.IsNullOrWhiteSpace(path))
 595                 throw new ArgumentNullException("path");
 596             if (!Path.IsPathRooted(path))
 597                 throw new ArgumentException("The path must be rooted", "path");
 598             Contract.EndContractBlock();
 599
 600             try
 601             {
 602
 603                 using (var connection = GetConnection())
 604                 using (var command = new SQLiteCommand("select OverlayStatus from FileState where FilePath=:path  COLLATE NOCASE", connection))
 605                 {
 606
 607                     command.Parameters.AddWithValue("path", path);
 608
 609                     var s = command.ExecuteScalar();
 610                     return (FileOverlayStatus) Convert.ToInt32(s);
 611                 }
 612             }
 613             catch (Exception exc)
 614             {
 615                 Log.ErrorFormat(exc.ToString());
 616                 return FileOverlayStatus.Unversioned;
 617             }
 618         }
 619
 620         private string GetConnectionString()
 621         {
 622             var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3;Enlist=N;Pooling=True", _pithosDataPath);
 623             return connectionString;
 624         }
 625
 626         private SQLiteConnection GetConnection()
 627         {
 628             var connectionString = GetConnectionString();
 629             var connection = new SQLiteConnection(connectionString);
 630             connection.Open();
 631             using(var cmd =connection.CreateCommand())
 632             {
 633                 cmd.CommandText = "PRAGMA journal_mode=WAL";
 634                 cmd.ExecuteNonQuery();
 635             }
 636             return connection;
 637         }
 638
 639        /* public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)
 640         {
 641             if (String.IsNullOrWhiteSpace(path))
 642                 throw new ArgumentNullException("path");
 643             if (!Path.IsPathRooted(path))
 644                 throw new ArgumentException("The path must be rooted","path");
 645             Contract.EndContractBlock();
 646
 647             _persistenceAgent.Post(() => FileState.StoreOverlayStatus(path,overlayStatus));
 648         }*/
 649
 650         public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string etag = null)
 651         {
 652             if (String.IsNullOrWhiteSpace(path))
 653                 throw new ArgumentNullException("path");
 654             if (!Path.IsPathRooted(path))
 655                 throw new ArgumentException("The path must be rooted","path");
 656             Contract.EndContractBlock();
 657
 658             return _persistenceAgent.PostAndAwait(() => FileState.StoreOverlayStatus(path,overlayStatus,etag));
 659         }
 660
 661        /* public void RenameFileOverlayStatus(string oldPath, string newPath)
 662         {
 663             if (String.IsNullOrWhiteSpace(oldPath))
 664                 throw new ArgumentNullException("oldPath");
 665             if (!Path.IsPathRooted(oldPath))
 666                 throw new ArgumentException("The oldPath must be rooted", "oldPath");
 667             if (String.IsNullOrWhiteSpace(newPath))
 668                 throw new ArgumentNullException("newPath");
 669             if (!Path.IsPathRooted(newPath))
 670                 throw new ArgumentException("The newPath must be rooted", "newPath");
 671             Contract.EndContractBlock();
 672
 673             _persistenceAgent.Post(() =>FileState.RenameState(oldPath, newPath));
 674         }*/
 675
 676         public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus, string conflictReason)
 677         {
 678             if (String.IsNullOrWhiteSpace(path))
 679                 throw new ArgumentNullException("path");
 680             if (!Path.IsPathRooted(path))
 681                 throw new ArgumentException("The path must be rooted", "path");
 682             Contract.EndContractBlock();
 683
 684             Debug.Assert(!path.Contains(FolderConstants.CacheFolder));
 685             Debug.Assert(!path.EndsWith(".ignore"));
 686
 687             _persistenceAgent.Post(() => UpdateStatusDirect(path, fileStatus, overlayStatus, conflictReason));
 688         }
 689
 690
 691         public void StoreInfo(string path, ObjectInfo objectInfo)
 692         {
 693             if (String.IsNullOrWhiteSpace(path))
 694                 throw new ArgumentNullException("path");
 695             if (!Path.IsPathRooted(path))
 696                 throw new ArgumentException("The path must be rooted", "path");
 697             if (objectInfo == null)
 698                 throw new ArgumentNullException("objectInfo", "objectInfo can't be empty");
 699             Contract.EndContractBlock();
 700
 701             _persistenceAgent.Post(() => StoreInfoDirect(path, objectInfo));
 702
 703         }
 704
 705         private void StoreInfoDirect(string path, ObjectInfo objectInfo)
 706         {
 707             try
 708             {
 709                     using (var session = ActiveRecordMediator.GetSessionFactoryHolder().CreateSession(typeof(FileState)))
 710                     using (var tx=session.BeginTransaction(IsolationLevel.ReadCommitted))
 711                     {
 712
 713                         //var walquery = session.CreateSQLQuery("PRAGMA journal_mode=WAL");
 714                         var walquery = session.CreateSQLQuery("PRAGMA journal_mode=WAL");
 715                         walquery.List();
 716
 717                         Func<IQuery, IQuery> setCriteria = q => q
 718                                                     .SetString("path", path)
 719                                                     .SetString("checksum",objectInfo.X_Object_Hash)
 720                                                     .SetString("etag", objectInfo.ETag)
 721                                                     .SetInt64("version", objectInfo.Version.GetValueOrDefault())
 722                                                     .SetDateTime("versionTimeStamp",objectInfo.VersionTimestamp.GetValueOrDefault())
 723                                                     .SetEnum("fileStatus", FileStatus.Unchanged)
 724                                                     .SetEnum("overlayStatus",FileOverlayStatus.Normal)
 725                                                     .SetString("objectID", objectInfo.UUID);
 726                         //IQuery updatecmd = session.CreateSQLQuery(
 727                         IQuery updatecmd = session.CreateQuery(
 728                             "update FileState set FilePath=:path,FileStatus= :fileStatus,OverlayStatus= :overlayStatus, Checksum=:checksum, ETag=:etag,LastMD5=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp where ObjectID = :objectID  ");
 729                         updatecmd = setCriteria(updatecmd);
 730                         var affected = updatecmd.ExecuteUpdate();
 731
 732                     //If the ID exists, update the status
 733                     if (affected == 0)
 734                     {
 735                         //If the ID doesn't exist, try to update using the path, and store the ID as well.
 736                         //updatecmd = session.CreateSQLQuery(
 737                         updatecmd = session.CreateQuery(
 738                         //    "update FileState set FileStatus= :fileStatus,OverlayStatus= :overlayStatus, ObjectID=:objectID, Checksum=:checksum, ETag=:etag,LastMD5=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp where FilePath = :path  COLLATE NOCASE ");
 739                             "update FileState set FileStatus= :fileStatus,OverlayStatus= :overlayStatus, ObjectID=:objectID, Checksum=:checksum, ETag=:etag,LastMD5=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp where FilePath = :path");
 740                         updatecmd=setCriteria(updatecmd);
 741                         affected = updatecmd.ExecuteUpdate();
 742                     }
 743                     if (affected==0)
 744                     {
 745                         //IQuery insertCmd=session.CreateSQLQuery(
 746                         IQuery insertCmd = session.CreateSQLQuery(
 747                             "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,ETag,LastMD5,FileStatus,OverlayStatus,ObjectID) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:etag,:etag,:fileStatus,:overlayStatus,:objectID)");
 748                         insertCmd=setCriteria(insertCmd).SetGuid("id", Guid.NewGuid());
 749                         affected = insertCmd.ExecuteUpdate();
 750                     }
 751                     tx.Commit();
 752                 }
 753             }
 754             catch (Exception exc)
 755             {
 756                 Log.ErrorFormat("Failed to update [{0}]:[{1}]\r\n{2}",path,objectInfo.UUID, exc);
 757                 throw;
 758             }
 759         }
 760
 761         private bool StateExists(string filePath,SQLiteConnection connection)
 762         {
 763             using (var command = new SQLiteCommand("Select count(*) from FileState where FilePath=:path  COLLATE NOCASE", connection))
 764             {
 765                 command.Parameters.AddWithValue("path", filePath);
 766                 var result = command.ExecuteScalar();
 767                 return ((long)result >= 1);
 768             }
 769
 770         }
 771
 772         private bool StateExistsByID(string objectId,SQLiteConnection connection)
 773         {
 774             using (var command = new SQLiteCommand("Select count(*) from FileState where ObjectId=:id", connection))
 775             {
 776                 command.Parameters.AddWithValue("id", objectId);
 777                 var result = command.ExecuteScalar();
 778                 return ((long)result >= 1);
 779             }
 780
 781         }
 782
 783         public void SetFileStatus(string path, FileStatus status)
 784         {
 785             if (String.IsNullOrWhiteSpace(path))
 786                 throw new ArgumentNullException("path");
 787             if (!Path.IsPathRooted(path))
 788                 throw new ArgumentException("The path must be rooted", "path");
 789             Contract.EndContractBlock();
 790
 791             _persistenceAgent.Post(() => UpdateStatusDirect(path, status));
 792         }
 793
 794         public FileStatus GetFileStatus(string path)
 795         {
 796             if (String.IsNullOrWhiteSpace(path))
 797                 throw new ArgumentNullException("path");
 798             if (!Path.IsPathRooted(path))
 799                 throw new ArgumentException("The path must be rooted", "path");
 800             Contract.EndContractBlock();
 801
 802
 803             using (var connection = GetConnection())
 804             {
 805                 var command = new SQLiteCommand("select FileStatus from FileState where FilePath=:path  COLLATE NOCASE", connection);
 806                 command.Parameters.AddWithValue("path", path);
 807
 808                 var statusValue = command.ExecuteScalar();
 809                 if (statusValue==null)
 810                     return FileStatus.Missing;
 811                 return (FileStatus)Convert.ToInt32(statusValue);
 812             }
 813         }
 814
 815         /// <summary>
 816         /// Deletes the status of the specified file
 817         /// </summary>
 818         /// <param name="path"></param>
 819         public void ClearFileStatus(string path)
 820         {
 821             if (String.IsNullOrWhiteSpace(path))
 822                 throw new ArgumentNullException("path");
 823             if (!Path.IsPathRooted(path))
 824                 throw new ArgumentException("The path must be rooted", "path");
 825             Contract.EndContractBlock();
 826
 827             _persistenceAgent.Post(() => DeleteDirect(path));
 828         }
 829
 830         /// <summary>
 831         /// Deletes the status of the specified folder and all its contents
 832         /// </summary>
 833         /// <param name="path"></param>
 834         public void ClearFolderStatus(string path)
 835         {
 836             if (String.IsNullOrWhiteSpace(path))
 837                 throw new ArgumentNullException("path");
 838             if (!Path.IsPathRooted(path))
 839                 throw new ArgumentException("The path must be rooted", "path");
 840             Contract.EndContractBlock();
 841             //TODO: May throw if the agent is cleared for some reason. Should never happen
 842             _persistenceAgent.Post(() => DeleteFolderDirect(path));
 843         }
 844
 845         public IEnumerable<FileState> GetChildren(FileState fileState)
 846         {
 847             if (fileState == null)
 848                 throw new ArgumentNullException("fileState");
 849             Contract.EndContractBlock();
 850
 851             var children = from state in FileState.Queryable
 852                            where state.FilePath.StartsWith(fileState.FilePath + "\\")
 853                            select state;
 854             return children;
 855         }
 856
 857         public void EnsureFileState(string path)
 858         {
 859             var existingState = GetStateByFilePath(path);
 860             if (existingState != null)
 861                 return;
 862             var fileInfo = FileInfoExtensions.FromPath(path);
 863             using (new SessionScope())
 864             {
 865                 var newState = FileState.CreateFor(fileInfo,StatusNotification);
 866                 newState.FileStatus=FileStatus.Missing;
 867                 _persistenceAgent.PostAndAwait(newState.CreateAndFlush).Wait();
 868             }
 869
 870         }
 871
 872         private int DeleteDirect(string filePath)
 873         {
 874             using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))
 875             {
 876
 877                 try
 878                 {
 879
 880
 881                     using (var connection = GetConnection())
 882                     {
 883                         var command = new SQLiteCommand("delete from FileState where FilePath = :path  COLLATE NOCASE",
 884                                                         connection);
 885
 886                         command.Parameters.AddWithValue("path", filePath);
 887
 888                         var affected = command.ExecuteNonQuery();
 889                         return affected;
 890                     }
 891                 }
 892                 catch (Exception exc)
 893                 {
 894                     Log.Error(exc.ToString());
 895                     throw;
 896                 }
 897             }
 898         }
 899
 900         private int DeleteFolderDirect(string filePath)
 901         {
 902             using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))
 903             {
 904
 905                 try
 906                 {
 907
 908
 909                     using (var connection = GetConnection())
 910                     {
 911                         var command = new SQLiteCommand(@"delete from FileState where FilePath = :path or FilePath like :path || '\%'  COLLATE NOCASE",
 912                                                         connection);
 913
 914                         command.Parameters.AddWithValue("path", filePath);
 915
 916                         var affected = command.ExecuteNonQuery();
 917                         return affected;
 918                     }
 919                 }
 920                 catch (Exception exc)
 921                 {
 922                     Log.Error(exc.ToString());
 923                     throw;
 924                 }
 925             }
 926         }
 927
 928         public void UpdateFileChecksum(string path, string etag, string checksum)
 929         {
 930             if (String.IsNullOrWhiteSpace(path))
 931                 throw new ArgumentNullException("path");
 932             if (!Path.IsPathRooted(path))
 933                 throw new ArgumentException("The path must be rooted", "path");
 934             Contract.EndContractBlock();
 935
 936             _persistenceAgent.Post(() => FileState.UpdateChecksum(path, etag,checksum));
 937         }
 938
 939         public void UpdateLastMD5(FileInfo file, string etag)
 940         {
 941             if (file==null)
 942                 throw new ArgumentNullException("file");
 943             if (String.IsNullOrWhiteSpace(etag))
 944                 throw new ArgumentNullException("etag");
 945             Contract.EndContractBlock();
 946
 947             _persistenceAgent.Post(() => FileState.UpdateLastMD5(file, etag));
 948         }
 949
 950
 951         public void CleanupOrphanStates()
 952         {
 953             //Orphan states are those that do not correspond to an account, ie. their paths
 954             //do not start with the root path of any registered account
 955
 956             var roots=(from account in Settings.Accounts
 957                       select account.RootPath).ToList();
 958
 959             var allStates = from state in FileState.Queryable
 960                 select state.FilePath;
 961
 962             foreach (var statePath in allStates)
 963             {
 964                 if (!roots.Any(root=>statePath.StartsWith(root,StringComparison.InvariantCultureIgnoreCase)))
 965                     this.DeleteDirect(statePath);
 966             }
 967         }
 968
 969         public void CleanupStaleStates(AccountInfo accountInfo, List<ObjectInfo> objectInfos)
 970         {
 971             if (accountInfo == null)
 972                 throw new ArgumentNullException("accountInfo");
 973             if (objectInfos == null)
 974                 throw new ArgumentNullException("objectInfos");
 975             Contract.EndContractBlock();
 976
 977
 978
 979             //Stale states are those that have no corresponding local or server file
 980
 981
 982             var agent=FileAgent.GetFileAgent(accountInfo);
 983
 984             var localFiles=agent.EnumerateFiles();
 985             var localSet = new HashSet<string>(localFiles);
 986
 987             //RelativeUrlToFilePath will fail for
 988             //infos of accounts, containers which have no Name
 989
 990             var serverFiles = from info in objectInfos
 991                               where info.Name != null
 992                               select Path.Combine(accountInfo.AccountPath,info.RelativeUrlToFilePath(accountInfo.UserName));
 993             var serverSet = new HashSet<string>(serverFiles);
 994
 995             var allStates = from state in FileState.Queryable
 996                             where state.FilePath.StartsWith(agent.RootPath)
 997                             select state.FilePath;
 998             var stateSet = new HashSet<string>(allStates);
 999             stateSet.ExceptWith(serverSet);
1000             stateSet.ExceptWith(localSet);
1001
1002             foreach (var remainder in stateSet)
1003             {
1004                 DeleteDirect(remainder);
1005             }
1006
1007
1008         }
1009     }
1010
1011
1012 }