d48053ed57352e05f295454b0ca581ae615ae178
[pithos-ms-client] / trunk / Pithos.Core / Agents / StatusAgent.cs
1 #region\r
2 /* -----------------------------------------------------------------------\r
3  * <copyright file="StatusAgent.cs" company="GRNet">\r
4  * \r
5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
6  *\r
7  * Redistribution and use in source and binary forms, with or\r
8  * without modification, are permitted provided that the following\r
9  * conditions are met:\r
10  *\r
11  *   1. Redistributions of source code must retain the above\r
12  *      copyright notice, this list of conditions and the following\r
13  *      disclaimer.\r
14  *\r
15  *   2. Redistributions in binary form must reproduce the above\r
16  *      copyright notice, this list of conditions and the following\r
17  *      disclaimer in the documentation and/or other materials\r
18  *      provided with the distribution.\r
19  *\r
20  *\r
21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
32  * POSSIBILITY OF SUCH DAMAGE.\r
33  *\r
34  * The views and conclusions contained in the software and\r
35  * documentation are those of the authors and should not be\r
36  * interpreted as representing official policies, either expressed\r
37  * or implied, of GRNET S.A.\r
38  * </copyright>\r
39  * -----------------------------------------------------------------------\r
40  */\r
41 #endregion\r
42 using System;\r
43 using System.Collections.Generic;\r
44 using System.ComponentModel.Composition;\r
45 using System.Data;\r
46 using System.Data.SqlServerCe;\r
47 using System.Diagnostics;\r
48 using System.Diagnostics.Contracts;\r
49 using System.IO;\r
50 using System.Linq;\r
51 using System.Reflection;\r
52 using System.Threading;\r
53 using NHibernate;\r
54 using NHibernate.Cfg;\r
55 using NHibernate.Cfg.MappingSchema;\r
56 using NHibernate.Criterion;\r
57 using NHibernate.Dialect;\r
58 using NHibernate.Linq;\r
59 using NHibernate.Mapping.ByCode;\r
60 using NHibernate.Tool.hbm2ddl;\r
61 using Pithos.Interfaces;\r
62 using Pithos.Network;\r
63 using log4net;\r
64 using Environment = System.Environment;\r
65 \r
66 namespace Pithos.Core.Agents\r
67 {\r
68     [Export(typeof(IStatusChecker)),Export(typeof(IStatusKeeper))]\r
69     public class StatusAgent:IStatusChecker,IStatusKeeper\r
70     {\r
71         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
72 \r
73         [System.ComponentModel.Composition.Import]\r
74         public IPithosSettings Settings { get; set; }\r
75 \r
76         [System.ComponentModel.Composition.Import]\r
77         public IStatusNotification StatusNotification { get; set; }\r
78 \r
79         //private Agent<Action> _persistenceAgent;\r
80 \r
81 \r
82 \r
83         public StatusAgent()\r
84         {\r
85             var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);\r
86 \r
87             _pithosDataPath = Path.Combine(appDataPath, "GRNET\\PITHOS");\r
88             if (!Directory.Exists(_pithosDataPath))\r
89                 Directory.CreateDirectory(_pithosDataPath);\r
90 \r
91             var dbPath = Path.Combine(_pithosDataPath, "pithos.sdf");\r
92 \r
93             //MigrateOldDb(dbPath, appDataPath);\r
94 \r
95 \r
96             var cfg = Configure(dbPath);\r
97 \r
98             var connectionString = "Data Source=" + dbPath;\r
99             using (var sqlCeEngine = new SqlCeEngine(connectionString))\r
100             {\r
101                 if (!File.Exists(dbPath))\r
102                 {\r
103                     sqlCeEngine.CreateDatabase();\r
104                     new SchemaExport(cfg).Execute(true, true, false);\r
105                     _factory = cfg.BuildSessionFactory();\r
106                     using (var session = _factory.OpenStatelessSession())\r
107                     {\r
108                         session.Insert(new PithosVersion {Id = 1, Version = "0.0.0.0"});\r
109                     }\r
110                 }\r
111                 else\r
112                 {\r
113                     try\r
114                     {\r
115                         if (!sqlCeEngine.Verify(VerifyOption.Enhanced))\r
116                             sqlCeEngine.Repair(connectionString, RepairOption.RecoverAllOrFail);\r
117                     }\r
118                     catch(SqlCeException ex)\r
119                     {\r
120                         //Rethrow except for sharing errors while repairing\r
121                         if (ex.NativeError != 25035)\r
122                             throw;\r
123                     }\r
124                     _factory = cfg.BuildSessionFactory();                    \r
125                 }\r
126                 UpgradeDatabase();\r
127             }\r
128         }\r
129 \r
130         private void UpgradeDatabase()\r
131         {\r
132             using (var session = _factory.OpenSession())\r
133             {\r
134 \r
135                 var storedVersion = session.Get<PithosVersion>(1);\r
136                 var actualVersion = Assembly.GetEntryAssembly().GetName().Version;\r
137 \r
138                 if (actualVersion == new Version(storedVersion.Version)) \r
139                     return;\r
140                 \r
141                 storedVersion.Version = actualVersion.ToString();\r
142                 session.Update(storedVersion);\r
143                 session.Flush();\r
144             }\r
145         }\r
146 \r
147 \r
148         private static Configuration Configure(string pithosDbPath)\r
149         {\r
150             if (String.IsNullOrWhiteSpace(pithosDbPath))\r
151                 throw new ArgumentNullException("pithosDbPath");\r
152             if (!Path.IsPathRooted(pithosDbPath))\r
153                 throw new ArgumentException("path must be a rooted path", "pithosDbPath");\r
154             Contract.EndContractBlock();\r
155 \r
156 \r
157             var cfg = new Configuration();                \r
158                 cfg.DataBaseIntegration(db=>\r
159                                          {\r
160                                              db.Dialect<MsSqlCe40Dialect>();\r
161                                              db.ConnectionString = "Data Source=" + pithosDbPath;\r
162                                              db.AutoCommentSql = true;\r
163                                              db.KeywordsAutoImport = Hbm2DDLKeyWords.AutoQuote;\r
164                                              db.SchemaAction = SchemaAutoAction.Update;\r
165                                              db.LogSqlInConsole = true;                                             \r
166                                          })                                                     \r
167                 .SessionFactory()                      \r
168                     .GenerateStatistics()                    \r
169                     .Integrate.Schema\r
170                         .Updating();            \r
171             var mapping = GetMapping();            \r
172             cfg.AddMapping(mapping);\r
173 \r
174             return cfg;\r
175         }\r
176 \r
177         private static HbmMapping GetMapping()\r
178         {\r
179             var mapper = new ModelMapper();\r
180             mapper.Class<FileState>(fm =>\r
181                                         {\r
182                                             fm.Id(x => x.Id, m => m.Generator(Generators.GuidComb));\r
183                                             fm.Property(x => x.ObjectID, m =>\r
184                                             {\r
185                                                 m.Unique(true);\r
186                                                 m.UniqueKey("U_FileState_ObjectID");\r
187                                                 m.Index("IX_FileState_ObjectID");\r
188                                             });\r
189                                             fm.Property(x => x.FilePath, m =>\r
190                                             {\r
191                                                 m.Unique(true);\r
192                                                 m.UniqueKey("U_FileState_FilePath");\r
193                                                 m.Index("IX_FileState_FilePath");                                                \r
194                                             });\r
195                                             fm.Property(x => x.OverlayStatus);\r
196                                             fm.Property(x => x.FileStatus);\r
197                                             fm.Property(x => x.ConflictReason);\r
198                                             fm.Property(x => x.Checksum, m => m.Length(64));\r
199                                             fm.Property(x => x.ETag, m => m.Length(64));\r
200                                             fm.Property(x => x.Hashes, m => m.Column(c => c.SqlType("ntext")));\r
201                                             fm.Property(x => x.LastWriteDate);\r
202                                             fm.Property(x => x.LastLength);\r
203                                             fm.Property(x => x.Version);\r
204                                             fm.Property(x => x.VersionTimeStamp);\r
205                                             fm.Property(x => x.IsShared);\r
206                                             fm.Property(x => x.SharedBy);\r
207                                             fm.Property(x => x.ShareWrite);\r
208                                             fm.Property(x => x.IsFolder);\r
209                                             //fm.Property(x => x.Modified);                                            \r
210                                         });\r
211             mapper.Class<PithosVersion>(fm =>\r
212                                         {\r
213                                             fm.Id(x => x.Id, m => m.Generator(Generators.Assigned));\r
214                                             fm.Property(x => x.Version, m => m.Length(20));\r
215                                         });\r
216 \r
217 \r
218             var mapping = mapper.CompileMappingFor(new[] {typeof (FileState),typeof(PithosVersion)});\r
219             return mapping;\r
220         }\r
221 \r
222         public void StartProcessing(CancellationToken token)\r
223         {\r
224            \r
225             \r
226         }\r
227 \r
228        \r
229 \r
230         public void Stop()\r
231         {\r
232           \r
233         }\r
234 \r
235 \r
236         public void ProcessExistingFiles(IEnumerable<FileInfo> existingFiles)\r
237         {\r
238             if (existingFiles == null)\r
239                 throw new ArgumentNullException("existingFiles");\r
240             Contract.EndContractBlock();\r
241 \r
242             //Find new or matching files with a left join to the stored states\r
243             using (var session = _factory.OpenSession())\r
244             {\r
245 \r
246                 var fileStates = session.Query<FileState>().ToList();\r
247                 var currentFiles = from file in existingFiles\r
248                                    join state in fileStates on file.FullName.ToLower() equals state.FilePath.ToLower()\r
249                                        into\r
250                                        gs\r
251                                    from substate in gs.DefaultIfEmpty()\r
252                                    select Tuple.Create(file, substate);\r
253 \r
254                 //To get the deleted files we must get the states that have no corresponding\r
255                 //files. \r
256                 //We can't use the File.Exists method inside a query, so we get all file paths from the states\r
257                 var statePaths = (from state in fileStates\r
258                                   select new {state.Id, state.FilePath}).ToList();\r
259                 //and check each one\r
260                 var missingStates = (from path in statePaths\r
261                                      where !File.Exists(path.FilePath) && !Directory.Exists(path.FilePath)\r
262                                      select path.Id).ToList();\r
263                 //Finally, retrieve the states that correspond to the deleted files            \r
264                 var deletedFiles = from state in fileStates\r
265                                    where missingStates.Contains(state.Id)\r
266                                    select Tuple.Create(default(FileInfo), state);\r
267 \r
268                 var pairs = currentFiles.Union(deletedFiles).ToList();\r
269 \r
270                 i = 1;\r
271                 var total = pairs.Count;\r
272                 foreach (var pair in pairs)\r
273                 {\r
274                     ProcessFile(session,total, pair);\r
275                 }\r
276                 session.Flush();\r
277             }\r
278         }\r
279 \r
280         int i = 1;\r
281 \r
282         private void ProcessFile(ISession session,int total, System.Tuple<FileInfo, FileState> pair)\r
283         {\r
284             var idx = Interlocked.Increment(ref i);\r
285             using (StatusNotification.GetNotifier("Indexing file {0} of {1}", "Indexed file {0} of {1} ", idx, total))\r
286             {\r
287                 var fileState = pair.Item2;\r
288                 var file = pair.Item1;\r
289                 if (fileState == null)\r
290                 {\r
291                     //This is a new file                        \r
292                     var createState = FileState.CreateFor(file,StatusNotification);                    \r
293                     session.Save(createState);\r
294                     //_persistenceAgent.Post(createState.Create);\r
295                 }\r
296                 else if (file == null)\r
297                 {\r
298                     //This file was deleted while we were down. We should mark it as deleted\r
299                     //We have to go through UpdateStatus here because the state object we are using\r
300                     //was created by a different ORM session.\r
301                     UpdateStatusDirect(session,fileState.Id, FileStatus.Deleted);\r
302                     //_persistenceAgent.Post(() => UpdateStatusDirect((Guid) fileState.Id, FileStatus.Deleted));\r
303                 }\r
304                 //else\r
305                 //{\r
306                 //    //This file has a matching state. Need to check for possible changes\r
307                 //    //To check for changes, we use the cheap (in CPU terms) MD5 algorithm\r
308                 //    //on the entire file.\r
309 \r
310                 //    var hashString = file.ComputeShortHash(StatusNotification);\r
311                 //    Debug.Assert(hashString.Length==32);\r
312 \r
313 \r
314                 //    //TODO: Need a way to attach the hashes to the filestate so we don't\r
315                 //    //recalculate them each time a call to calculate has is made\r
316                 //    //We can either store them to the filestate or add them to a \r
317                 //    //dictionary\r
318 \r
319                 //    //If the hashes don't match the file was changed\r
320                 //    if (fileState.ETag != hashString)\r
321                 //    {\r
322                 //        _persistenceAgent.Post(() => UpdateStatusDirect((Guid) fileState.Id, FileStatus.Modified));\r
323                 //    }\r
324                 //}\r
325             }\r
326         }\r
327 \r
328 \r
329         private int UpdateStatusDirect(ISession session,Guid id, FileStatus status)\r
330         {\r
331             using (ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))\r
332             {\r
333 \r
334                 try\r
335                 {\r
336                         //var updatecmd = session.CreateSQLQuery(\r
337                     var updatecmd = session.CreateQuery(\r
338                         "update FileState set FileStatus= :fileStatus where Id = :id  ")\r
339                         .SetGuid("id", id)\r
340                         .SetEnum("fileStatus", status);\r
341                     var affected = updatecmd.ExecuteUpdate();\r
342                         session.Flush();\r
343                     return affected;\r
344                 }\r
345                 catch (Exception exc)\r
346                 {\r
347                     Log.Error(exc.ToString());\r
348                     throw;\r
349                 }\r
350             }\r
351         }\r
352 \r
353 \r
354         public string BlockHash { get; set; }\r
355 \r
356         public int BlockSize { get; set; }\r
357 \r
358         public void ChangeRoots(string oldPath, string newPath)\r
359         {\r
360             if (String.IsNullOrWhiteSpace(oldPath))\r
361                 throw new ArgumentNullException("oldPath");\r
362             if (!Path.IsPathRooted(oldPath))\r
363                 throw new ArgumentException("oldPath must be an absolute path", "oldPath");\r
364             if (String.IsNullOrWhiteSpace(newPath))\r
365                 throw new ArgumentNullException("newPath");\r
366             if (!Path.IsPathRooted(newPath))\r
367                 throw new ArgumentException("newPath must be an absolute path", "newPath");\r
368             Contract.EndContractBlock();\r
369 \r
370             ChangeRootPath(oldPath,newPath);\r
371 \r
372         }\r
373 \r
374 \r
375 \r
376         private readonly string _pithosDataPath;\r
377         private readonly ISessionFactory _factory;\r
378 \r
379         public FileState GetStateByFilePath(string path)\r
380         {\r
381             if (String.IsNullOrWhiteSpace(path))\r
382                 throw new ArgumentNullException("path");\r
383             if (!Path.IsPathRooted(path))\r
384                 throw new ArgumentException("The path must be rooted", "path");\r
385             Contract.EndContractBlock();\r
386 \r
387             try\r
388             {\r
389                 \r
390                 using(var session=_factory.OpenStatelessSession())\r
391                 {\r
392                     var state=session.Query<FileState>().SingleOrDefault(s => s.FilePath == path);\r
393                     if (state==null)\r
394                         return null;\r
395                     state.FilePath=state.FilePath??String.Empty;\r
396                     state.OverlayStatus = state.OverlayStatus ??FileOverlayStatus.Unversioned;\r
397                     state.FileStatus = state.FileStatus ?? FileStatus.Missing;\r
398                     state.Checksum = state.Checksum ?? String.Empty;\r
399                     state.ETag = state.ETag ?? String.Empty;\r
400                     state.SharedBy = state.SharedBy ?? String.Empty;\r
401                     return state;\r
402                 }\r
403 \r
404             }\r
405             catch (Exception exc)\r
406             {\r
407                 Log.ErrorFormat(exc.ToString());\r
408                 throw;\r
409             }            \r
410         }\r
411 \r
412         public FileOverlayStatus GetFileOverlayStatus(string path)\r
413         {\r
414             if (String.IsNullOrWhiteSpace(path))\r
415                 throw new ArgumentNullException("path");\r
416             if (!Path.IsPathRooted(path))\r
417                 throw new ArgumentException("The path must be rooted", "path");\r
418             Contract.EndContractBlock();\r
419 \r
420             try\r
421             {\r
422                 \r
423                 using(var session=_factory.OpenStatelessSession())\r
424                 {\r
425                     return (from state in session.Query<FileState>()\r
426                             where state.FilePath == path\r
427                             select state.OverlayStatus)\r
428                             .Single()\r
429                             .GetValueOrDefault(FileOverlayStatus.Unversioned);\r
430                 }\r
431             }\r
432             catch (Exception exc)\r
433             {\r
434                 Log.ErrorFormat(exc.ToString());\r
435                 return FileOverlayStatus.Unversioned;\r
436             }\r
437         }\r
438 \r
439         public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)\r
440         {\r
441             if (String.IsNullOrWhiteSpace(path))\r
442                 throw new ArgumentNullException("path");\r
443             if (!Path.IsPathRooted(path))\r
444                 throw new ArgumentException("The path must be rooted","path");\r
445             Contract.EndContractBlock();\r
446 \r
447             StoreOverlayStatus(path,overlayStatus);\r
448         }\r
449 \r
450         public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus, string conflictReason)\r
451         {\r
452             if (String.IsNullOrWhiteSpace(path))\r
453                 throw new ArgumentNullException("path");\r
454             if (!Path.IsPathRooted(path))\r
455                 throw new ArgumentException("The path must be rooted", "path");\r
456             Contract.EndContractBlock();\r
457 \r
458             Debug.Assert(!path.Contains(FolderConstants.CacheFolder));\r
459             Debug.Assert(!path.EndsWith(".ignore"));            \r
460             using (ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))\r
461             {\r
462 \r
463                 try\r
464                 {\r
465 \r
466                     using (var session = _factory.OpenSession())\r
467                     using (var tx=session.BeginTransaction(IsolationLevel.ReadCommitted))\r
468                     {\r
469 \r
470                         //var updatecmd = session.CreateSQLQuery("update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus,ConflictReason= :conflictReason where FilePath = :path ")\r
471                         var updatecmd = session.CreateQuery("update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus,ConflictReason= :conflictReason where FilePath = :path")\r
472                             .SetString("path", path)\r
473                             .SetEnum("fileStatus", fileStatus)\r
474                             .SetEnum("overlayStatus", overlayStatus)\r
475                             .SetString("conflictReason", conflictReason);\r
476                         var affected = updatecmd.ExecuteUpdate();\r
477 \r
478                         if (affected == 0)\r
479                         {\r
480                             //Can happen when downloading a new file\r
481                             var createdState = FileState.CreateFor(FileInfoExtensions.FromPath(path), StatusNotification);\r
482                             createdState.FileStatus = fileStatus;\r
483                             createdState.OverlayStatus = overlayStatus;                            \r
484                             createdState.ConflictReason = conflictReason;\r
485                             session.Save(createdState);\r
486                             //createdState.Create();\r
487                         }\r
488                         session.Flush();\r
489                         tx.Commit();                        \r
490                     }\r
491                 }\r
492                 catch (Exception exc)\r
493                 {\r
494                     Log.Error(exc.ToString());\r
495                     throw;\r
496                 }\r
497             }            \r
498         }\r
499 \r
500 \r
501         public void StoreInfo(string path, ObjectInfo objectInfo, TreeHash treeHash)\r
502         {\r
503             if (String.IsNullOrWhiteSpace(path))\r
504                 throw new ArgumentNullException("path");\r
505             if (treeHash==null)\r
506                 throw new ArgumentNullException("treeHash");\r
507             if (!Path.IsPathRooted(path))\r
508                 throw new ArgumentException("The path must be rooted", "path");\r
509             if (objectInfo == null)\r
510                 throw new ArgumentNullException("objectInfo", "objectInfo can't be empty");\r
511             Contract.EndContractBlock();\r
512 \r
513             StoreInfoDirect(path, objectInfo, treeHash);\r
514 \r
515         }\r
516 \r
517         public void StoreInfo(string path, ObjectInfo objectInfo)\r
518         {\r
519             if (String.IsNullOrWhiteSpace(path))\r
520                 throw new ArgumentNullException("path");\r
521             if (!Path.IsPathRooted(path))\r
522                 throw new ArgumentException("The path must be rooted", "path");\r
523             if (objectInfo == null)\r
524                 throw new ArgumentNullException("objectInfo", "objectInfo can't be empty");\r
525             Contract.EndContractBlock();\r
526 \r
527             StoreInfoDirect(path, objectInfo, null);\r
528 \r
529         }\r
530 \r
531         private void StoreInfoDirect(string path, ObjectInfo objectInfo,TreeHash treeHash)\r
532         {\r
533             try\r
534             {\r
535                     using (var session = _factory.OpenSession())\r
536                     using (var tx=session.BeginTransaction(IsolationLevel.ReadCommitted))\r
537                     {\r
538                         \r
539                         //An entry for the new path may exist, \r
540                         IQuery deletecmd = session.CreateQuery(\r
541                            "delete from FileState where FilePath=:path and ObjectID is null")\r
542                            .SetString("path",path);                        \r
543                         deletecmd.ExecuteUpdate();\r
544 \r
545                         //string md5=treeHash.NullSafe(t=>t.MD5);                        \r
546                         string hashes=treeHash.NullSafe(t=>t.ToJson());\r
547 \r
548                         var info = FileInfoExtensions.FromPath(path);\r
549                         var lastWriteTime = info.LastWriteTime;\r
550                         var isFolder = (info is DirectoryInfo);\r
551                         var lastLength=isFolder ? 0:((FileInfo) info).Length;\r
552 \r
553                         Func<IQuery, IQuery> setCriteria = q => {\r
554                                 var q1=q.SetString("path", path)\r
555                                     .SetBoolean("isFolder",isFolder)\r
556                                     .SetDateTime("lastWrite",lastWriteTime)\r
557                                     .SetInt64("lastLength",lastLength)\r
558                                     .SetString("checksum", objectInfo.X_Object_Hash)\r
559                                     .SetString("etag", objectInfo.ETag)\r
560                                     .SetInt64("version",objectInfo.Version.GetValueOrDefault())\r
561                                     .SetDateTime("versionTimeStamp",objectInfo.VersionTimestamp.GetValueOrDefault())\r
562                                     .SetEnum("fileStatus", FileStatus.Unchanged)\r
563                                     .SetEnum("overlayStatus",FileOverlayStatus.Normal)\r
564                                     .SetString("objectID", objectInfo.UUID);\r
565                                 if (treeHash!=null)\r
566                                 {\r
567                                     q1 = q1.SetString("hashes", hashes);\r
568                                       //  .SetString("md5", md5);     \r
569                                 }\r
570                                 return q1;\r
571                         };\r
572 \r
573                     var updateStatement=(treeHash!=null)\r
574                         ? "update FileState set FilePath=:path,IsFolder=:isFolder,LastWriteDate=:lastWrite,LastLength=:lastLength,FileStatus= :fileStatus,OverlayStatus= :overlayStatus, Checksum=:checksum,Hashes=:hashes, ETag=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp "\r
575                         : "update FileState set FilePath=:path,IsFolder=:isFolder,LastWriteDate=:lastWrite,LastLength=:lastLength,FileStatus= :fileStatus,OverlayStatus= :overlayStatus, Checksum=:checksum, ETag=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp ";\r
576                         //? "update FileState set FilePath=:path,IsFolder=:isFolder,LastWriteDate=:lastWrite,LastLength=:lastLength,FileStatus= :fileStatus,OverlayStatus= :overlayStatus, Checksum=:checksum,Hashes=:hashes, ETag=:etag,LastMD5=:md5,Version=:version,VersionTimeStamp=:versionTimeStamp "\r
577                         //: "update FileState set FilePath=:path,IsFolder=:isFolder,LastWriteDate=:lastWrite,LastLength=:lastLength,FileStatus= :fileStatus,OverlayStatus= :overlayStatus, Checksum=:checksum, ETag=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp ";\r
578 \r
579                     IQuery updatecmd = session.CreateQuery(updateStatement + " where ObjectID = :objectID  ");\r
580 \r
581 \r
582                     updatecmd = setCriteria(updatecmd);\r
583                     //If the ID exists, update the status                                          \r
584                     var affected = updatecmd.ExecuteUpdate();                        \r
585                         \r
586                     //If the ID doesn't exist, try to update using the path, and store the ID as well.\r
587                     if (affected == 0)\r
588                     {\r
589                         updateStatement=(treeHash!=null)\r
590                             ? "update FileState set IsFolder=:isFolder,LastWriteDate=:lastWrite,LastLength=:lastLength,FileStatus= :fileStatus,OverlayStatus= :overlayStatus, ObjectID=:objectID, Checksum=:checksum,Hashes=:hashes, ETag=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp "\r
591                             : "update FileState set IsFolder=:isFolder,LastWriteDate=:lastWrite,LastLength=:lastLength,FileStatus= :fileStatus,OverlayStatus= :overlayStatus, ObjectID=:objectID, Checksum=:checksum, ETag=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp ";\r
592                             //? "update FileState set IsFolder=:isFolder,LastWriteDate=:lastWrite,LastLength=:lastLength,FileStatus= :fileStatus,OverlayStatus= :overlayStatus, ObjectID=:objectID, Checksum=:checksum,Hashes=:hashes, ETag=:etag,LastMD5=:md5,Version=:version,VersionTimeStamp=:versionTimeStamp "\r
593                             //: "update FileState set IsFolder=:isFolder,LastWriteDate=:lastWrite,LastLength=:lastLength,FileStatus= :fileStatus,OverlayStatus= :overlayStatus, ObjectID=:objectID, Checksum=:checksum, ETag=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp ";\r
594                         updatecmd = session.CreateQuery(updateStatement + " where FilePath = :path");\r
595                         updatecmd=setCriteria(updatecmd);\r
596                         affected = updatecmd.ExecuteUpdate();\r
597                     }\r
598                     //If the record can't be located, create a new one\r
599                     if (affected==0)\r
600                     {                        \r
601                         IQuery insertCmd = session.CreateSQLQuery("INSERT INTO FileState (Id,FilePath,IsFolder,LastWriteDate,LastLength,Checksum,Hashes,Version,VersionTimeStamp,ETag,FileStatus,OverlayStatus,ObjectID) " + \r
602                             "VALUES (:id,:path,:isFolder,:lastWrite,:lastLength,:checksum,:hashes,:version,:versionTimeStamp,:etag,:fileStatus,:overlayStatus,:objectID)");\r
603                         insertCmd=setCriteria(insertCmd)\r
604                             .SetGuid("id", Guid.NewGuid());\r
605                         affected = insertCmd.ExecuteUpdate();\r
606                     }\r
607                     session.Flush();\r
608                     tx.Commit();\r
609                 }\r
610             }\r
611             catch (Exception exc)\r
612             {\r
613                 Log.ErrorFormat("Failed to update [{0}]:[{1}]\r\n{2}",path,objectInfo.UUID, exc);\r
614                 throw;\r
615             }\r
616         }\r
617 \r
618 \r
619 \r
620         public void SetFileStatus(string path, FileStatus status)\r
621         {\r
622             if (String.IsNullOrWhiteSpace(path))\r
623                 throw new ArgumentNullException("path");\r
624             if (!Path.IsPathRooted(path))\r
625                 throw new ArgumentException("The path must be rooted", "path");\r
626             Contract.EndContractBlock();\r
627 \r
628             using (ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))\r
629             {\r
630 \r
631                 try\r
632                 {                    \r
633                     using (var session = _factory.OpenSession())\r
634                     using (var tx=session.BeginTransaction(IsolationLevel.ReadCommitted))\r
635                     {\r
636 \r
637                         //var updatecmd = session.CreateSQLQuery(\r
638                         var updatecmd = session.CreateQuery(\r
639                             "update FileState set FileStatus= :fileStatus where FilePath = :path ")\r
640                             .SetString("path", path)\r
641                             .SetEnum("fileStatus", status);\r
642                         var affected = updatecmd.ExecuteUpdate();\r
643 \r
644                         if (affected == 0)\r
645                         {                            \r
646                             var createdState = FileState.CreateFor(FileInfoExtensions.FromPath(path), StatusNotification);\r
647                             createdState.FileStatus = status;\r
648                             session.Save(createdState);\r
649                         }\r
650                         session.Flush();\r
651                         tx.Commit();\r
652                     }\r
653                 }\r
654                 catch (Exception exc)\r
655                 {\r
656                     Log.Error(exc.ToString());\r
657                     throw;\r
658                 }\r
659             }            \r
660         }\r
661 \r
662         public FileStatus GetFileStatus(string path)\r
663         {\r
664             if (String.IsNullOrWhiteSpace(path))\r
665                 throw new ArgumentNullException("path");\r
666             if (!Path.IsPathRooted(path))\r
667                 throw new ArgumentException("The path must be rooted", "path");\r
668             Contract.EndContractBlock();\r
669 \r
670             \r
671             using(var session=_factory.OpenStatelessSession())\r
672                 return (from state in session.Query<FileState>()\r
673                         select state.FileStatus).SingleOrDefault()??FileStatus.Missing;\r
674         }\r
675 \r
676         /// <summary>\r
677         /// Deletes the status of the specified file\r
678         /// </summary>\r
679         /// <param name="path"></param>\r
680         public void ClearFileStatus(string path)\r
681         {\r
682             if (String.IsNullOrWhiteSpace(path))\r
683                 throw new ArgumentNullException("path");\r
684             if (!Path.IsPathRooted(path))\r
685                 throw new ArgumentException("The path must be rooted", "path");\r
686             Contract.EndContractBlock();\r
687             using(var session=_factory.OpenSession())\r
688             {\r
689                 DeleteDirect(session,path);\r
690                 session.Flush();                \r
691             }\r
692         }\r
693 \r
694         /// <summary>\r
695         /// Deletes the status of the specified folder and all its contents\r
696         /// </summary>\r
697         /// <param name="path"></param>\r
698         public void ClearFolderStatus(string path)\r
699         {\r
700             if (String.IsNullOrWhiteSpace(path))\r
701                 throw new ArgumentNullException("path");\r
702             if (!Path.IsPathRooted(path))\r
703                 throw new ArgumentException("The path must be rooted", "path");\r
704             Contract.EndContractBlock();\r
705             using (ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))\r
706             {\r
707 \r
708                 try\r
709                 {\r
710                     using (var session = _factory.OpenSession())\r
711                     {\r
712                         DeleteDirect(session,path);\r
713                         session.Flush();\r
714                     }\r
715                 }\r
716                 catch (Exception exc)\r
717                 {\r
718                     Log.Error(exc.ToString());\r
719                     throw;\r
720                 }\r
721             }            \r
722         }\r
723 \r
724         public IEnumerable<FileState> GetChildren(FileState fileState)\r
725         {\r
726             if (fileState == null)\r
727                 throw new ArgumentNullException("fileState");\r
728             Contract.EndContractBlock();\r
729 \r
730             var session = _factory.GetCurrentSession();\r
731             var children = from state in session.Query<FileState>()\r
732                            where state.FilePath.StartsWith(fileState.FilePath + "\\")\r
733                            select state;\r
734             return children;\r
735         }\r
736 \r
737         public void EnsureFileState(string path)\r
738         {\r
739             var existingState = GetStateByFilePath(path);\r
740             if (existingState != null)\r
741                 return;\r
742             var fileInfo = FileInfoExtensions.FromPath(path);\r
743             \r
744             using (var session=_factory.OpenSession())\r
745             {\r
746                 var newState = FileState.CreateFor(fileInfo,StatusNotification);\r
747                 newState.FileStatus=FileStatus.Missing;\r
748                 session.SaveOrUpdate(newState);\r
749                 session.Flush();\r
750                 //_persistenceAgent.PostAndAwait(newState.CreateAndFlush).Wait();\r
751             }\r
752 \r
753         }\r
754 \r
755         private void DeleteDirect(ISession session,string filePath)\r
756         {\r
757             using (ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))\r
758             {\r
759 \r
760                 try\r
761                 {\r
762                     var deletes= session.CreateQuery("delete from FileState where FilePath = :path")\r
763                         .SetParameter("path", filePath)\r
764                         .ExecuteUpdate();                    \r
765                 }\r
766                 catch (Exception exc)\r
767                 {\r
768                     Log.Error(exc.ToString());\r
769                     throw;\r
770                 }\r
771             }\r
772         }\r
773 \r
774 \r
775         public void UpdateFileChecksum(string path, string etag, TreeHash treeHash)\r
776         {\r
777             if (String.IsNullOrWhiteSpace(path))\r
778                 throw new ArgumentNullException("path");\r
779             if (!Path.IsPathRooted(path))\r
780                 throw new ArgumentException("The path must be rooted", "path");            \r
781             Contract.EndContractBlock();\r
782 \r
783             UpdateChecksum(path, etag,treeHash);\r
784         }\r
785 \r
786 \r
787         public void CleanupOrphanStates()\r
788         {\r
789             //Orphan states are those that do not correspond to an account, ie. their paths\r
790             //do not start with the root path of any registered account\r
791 \r
792             var roots=(from account in Settings.Accounts\r
793                       select account.RootPath).ToList();\r
794 \r
795             using (var session = _factory.OpenSession())\r
796             {\r
797                 var allStates = from state in session.Query<FileState>()\r
798                                 select state.FilePath;\r
799 \r
800                 foreach (var statePath in allStates)\r
801                 {\r
802                     if (!roots.Any(root => statePath.StartsWith(root, StringComparison.InvariantCultureIgnoreCase)))\r
803                         this.DeleteDirect(session,statePath);\r
804                 }\r
805                 session.Flush();\r
806             }\r
807         }\r
808 \r
809         public void SaveCopy<T>(T state) where T:class\r
810         {\r
811             using (var session = _factory.OpenSession())\r
812             {\r
813                 session.Merge(state);\r
814                 session.Flush();\r
815             }\r
816         }\r
817 \r
818         public void CleanupStaleStates(AccountInfo accountInfo, List<ObjectInfo> objectInfos)\r
819         {\r
820             if (accountInfo == null)\r
821                 throw new ArgumentNullException("accountInfo");\r
822             if (objectInfos == null)\r
823                 throw new ArgumentNullException("objectInfos");\r
824             Contract.EndContractBlock();\r
825             \r
826 \r
827 \r
828             //Stale states are those that have no corresponding local or server file\r
829             \r
830 \r
831             var agent=FileAgent.GetFileAgent(accountInfo);\r
832 \r
833             var localFiles=agent.EnumerateFiles();\r
834             var localSet = new HashSet<string>(localFiles);\r
835 \r
836             //RelativeUrlToFilePath will fail for\r
837             //infos of accounts, containers which have no Name\r
838 \r
839             var serverFiles = from info in objectInfos\r
840                               where info.Name != null\r
841                               select Path.Combine(accountInfo.AccountPath,info.RelativeUrlToFilePath(accountInfo.UserName));\r
842             var serverSet = new HashSet<string>(serverFiles);\r
843 \r
844             using (var session = _factory.OpenSession())\r
845             {\r
846 \r
847                 var allStates = from state in session.Query<FileState>()\r
848                                 where state.FilePath.StartsWith(agent.RootPath)\r
849                                 select state.FilePath;\r
850                 var stateSet = new HashSet<string>(allStates);\r
851                 stateSet.ExceptWith(serverSet);\r
852                 stateSet.ExceptWith(localSet);\r
853 \r
854                 foreach (var remainder in stateSet)\r
855                 {\r
856                     DeleteDirect(session,remainder);\r
857                 }\r
858                 session.Flush();\r
859             }\r
860         }\r
861 \r
862         public static TreeHash CalculateTreeHash(FileSystemInfo fileInfo, AccountInfo accountInfo, FileState fileState, byte hashingParallelism, CancellationToken cancellationToken, Progress<double> progress)\r
863         {\r
864             fileInfo.Refresh();\r
865             //If the file doesn't exist, return the empty treehash\r
866             if (!fileInfo.Exists)\r
867                 return TreeHash.Empty;\r
868 \r
869             //FileState may be null if there is no stored state for this file\r
870             //if (fileState==null)\r
871                 return Signature.CalculateTreeHashAsync(fileInfo,\r
872                                                  accountInfo.BlockSize,\r
873                                                  accountInfo.BlockHash,\r
874                                                  hashingParallelism,\r
875                                                  cancellationToken, progress);\r
876             //Can we use the stored hashes?\r
877             //var localTreeHash = fileState.LastMD5 == Signature.CalculateMD5(fileInfo)\r
878             //                        ? TreeHash.Parse(fileState.Hashes)\r
879             //                        : Signature.CalculateTreeHashAsync(fileInfo,\r
880             //                                                           accountInfo.BlockSize,\r
881             //                                                           accountInfo.BlockHash,\r
882             //                                                           hashingParallelism,\r
883             //                                                           cancellationToken, progress);\r
884             //return localTreeHash;\r
885         }\r
886 \r
887 \r
888 \r
889         private object ExecuteWithRetry(Func<ISession, object, object> call, object state)\r
890         {\r
891             int retries = 3;\r
892             while (retries > 0)\r
893                 try\r
894                 {\r
895                     using (var session=_factory.OpenSession())\r
896                     {\r
897                         var result=call(session, state);\r
898                         session.Flush();\r
899                         return result;\r
900                     }\r
901                 }\r
902                 catch (Exception/* ActiveRecordException */)\r
903                 {\r
904                     retries--;\r
905                     if (retries <= 0)\r
906                         throw;\r
907                 }\r
908             return null;\r
909         }\r
910 \r
911         //TODO: Must separate between UpdateChecksum and UpdateFileTreeHash\r
912         public  void UpdateChecksum(string absolutePath, string etag, TreeHash treeHash)\r
913         {\r
914             if (string.IsNullOrWhiteSpace(absolutePath))\r
915                 throw new ArgumentNullException("absolutePath");\r
916             Contract.EndContractBlock();\r
917 \r
918             var hashes = treeHash.ToJson();\r
919             var topHash = treeHash.TopHash.ToHashString();\r
920 \r
921             ExecuteWithRetry((session, instance) =>\r
922             {\r
923                 const string hqlUpdate = "update FileState set Checksum= :checksum,Hashes=:hashes,ETag=:etag where FilePath = :path ";\r
924                 var updatedEntities = session.CreateQuery(hqlUpdate)\r
925                     .SetString("path", absolutePath)\r
926                     .SetString("checksum", topHash)\r
927                     .SetString("hashes", hashes)\r
928                     .SetString("etag", etag)\r
929                     .ExecuteUpdate();\r
930                 return updatedEntities;\r
931             }, null);\r
932 \r
933         }\r
934 \r
935         public  void ChangeRootPath(string oldPath, string newPath)\r
936         {\r
937             if (String.IsNullOrWhiteSpace(oldPath))\r
938                 throw new ArgumentNullException("oldPath");\r
939             if (!Path.IsPathRooted(oldPath))\r
940                 throw new ArgumentException("oldPath must be an absolute path", "oldPath");\r
941             if (string.IsNullOrWhiteSpace(newPath))\r
942                 throw new ArgumentNullException("newPath");\r
943             if (!Path.IsPathRooted(newPath))\r
944                 throw new ArgumentException("newPath must be an absolute path", "newPath");\r
945             Contract.EndContractBlock();\r
946 \r
947             //Ensure the paths end with the same character\r
948             if (!oldPath.EndsWith("\\"))\r
949                 oldPath = oldPath + "\\";\r
950             if (!newPath.EndsWith("\\"))\r
951                 newPath = newPath + "\\";\r
952 \r
953             ExecuteWithRetry((session, instance) =>\r
954             {\r
955                 const string hqlUpdate =\r
956                     "update FileState set FilePath = replace(FilePath,:oldPath,:newPath) where FilePath like :oldPath || '%' ";\r
957                 var renames = session.CreateQuery(hqlUpdate)\r
958                     .SetString("oldPath", oldPath)\r
959                     .SetString("newPath", newPath)\r
960                     .ExecuteUpdate();\r
961                 return renames;\r
962             }, null);\r
963         }\r
964 \r
965 \r
966         /// <summary>\r
967         /// Mark Unversioned all FileState rows from the database whose path\r
968         /// starts with one of the removed paths\r
969         /// </summary>\r
970         /// <param name="removed"></param>\r
971         public void UnversionPaths(List<string> removed)\r
972         {\r
973             if (removed == null)\r
974                 return;\r
975             if (removed.Count == 0)\r
976                 return;\r
977 \r
978             //Create a disjunction (list of OR statements\r
979             var disjunction = new Disjunction();\r
980             foreach (var path in removed)\r
981             {\r
982                 //with the restriction FileState.FilePath like '@path%'\r
983                 disjunction.Add(Restrictions.On<FileState>(s => s.FilePath)\r
984                     .IsLike(path, MatchMode.Start));\r
985             }\r
986 \r
987             //Generate a query from the disjunction\r
988             var query = QueryOver.Of<FileState>().Where(disjunction);\r
989 \r
990             ExecuteWithRetry((session, instance) =>\r
991             {\r
992                 using (var tx = session.BeginTransaction())\r
993                 {\r
994                     var states = query.GetExecutableQueryOver(session).List();\r
995                     foreach (var state in states)\r
996                     {\r
997                         state.FileStatus = FileStatus.Unversioned;\r
998                         state.OverlayStatus = FileOverlayStatus.Unversioned;\r
999                         session.Update(session);\r
1000                     }\r
1001                     tx.Commit();\r
1002                 }\r
1003                 return null;\r
1004             }, null);\r
1005         }\r
1006 \r
1007         public List<FileState> GetAllStates()\r
1008         {\r
1009             using(var session=_factory.OpenSession())\r
1010             {\r
1011                 return session.Query<FileState>().ToList();\r
1012             }\r
1013         }\r
1014 \r
1015         public List<string> GetAllStatePaths()\r
1016         {\r
1017             using (var session = _factory.OpenSession())\r
1018             {\r
1019                 return session.Query<FileState>().Select(state => state.FilePath).ToList();\r
1020             }\r
1021         }\r
1022 \r
1023         public List<FileState> GetConflictStates()\r
1024         {\r
1025             using (var session = _factory.OpenSession())\r
1026             {\r
1027                 var fileStates = from state in session.Query<FileState>()\r
1028                                  where state.FileStatus == FileStatus.Conflict ||\r
1029                                        state.OverlayStatus == FileOverlayStatus.Conflict\r
1030                                  select state;\r
1031                 return fileStates.ToList();\r
1032             }\r
1033         }\r
1034 \r
1035 \r
1036         public  void UpdateFileTreeHash(string absolutePath, TreeHash treeHash)\r
1037         {\r
1038             if (string.IsNullOrWhiteSpace(absolutePath))\r
1039                 throw new ArgumentNullException("absolutePath");\r
1040             Contract.EndContractBlock();\r
1041 \r
1042             var hashes = treeHash.ToJson();\r
1043             var topHash = treeHash.TopHash.ToHashString();\r
1044 \r
1045             ExecuteWithRetry((session, instance) =>\r
1046             {\r
1047 \r
1048                 const string hqlUpdate = "update FileState set Checksum= :checksum,Hashes=:hashes where FilePath = :path ";\r
1049                 var updatedEntities = session.CreateQuery(hqlUpdate)\r
1050                     .SetString("path", absolutePath)\r
1051                     .SetString("checksum", topHash)\r
1052                     //                    .SetString("md5",treeHash.MD5)\r
1053                     .SetString("hashes", hashes)\r
1054                     .ExecuteUpdate();\r
1055                 return updatedEntities;\r
1056             }, null);\r
1057         }\r
1058 \r
1059 \r
1060         public void RenameState(string oldPath, string newPath)\r
1061         {\r
1062             if (string.IsNullOrWhiteSpace(oldPath))\r
1063                 throw new ArgumentNullException("oldPath");\r
1064             Contract.EndContractBlock();\r
1065 \r
1066             ExecuteWithRetry((session, instance) =>\r
1067             {\r
1068                 const string hqlUpdate =\r
1069                     "update FileState set FilePath= :newPath where FilePath = :oldPath ";\r
1070                 var updatedEntities = session.CreateQuery(hqlUpdate)\r
1071                     .SetString("oldPath", oldPath)\r
1072                     .SetString("newPath", newPath)\r
1073                     .ExecuteUpdate();\r
1074                 return updatedEntities;\r
1075             }, null);\r
1076 \r
1077         }\r
1078 \r
1079         public void StoreOverlayStatus(string absolutePath, FileOverlayStatus newStatus)\r
1080         {\r
1081             if (string.IsNullOrWhiteSpace(absolutePath))\r
1082                 throw new ArgumentNullException("absolutePath");\r
1083             Contract.EndContractBlock();\r
1084 \r
1085             ExecuteWithRetry((session, instance) =>\r
1086             {\r
1087                 const string hqlUpdate =\r
1088                     "update FileState set OverlayStatus= :status where FilePath = :path ";\r
1089                 using (var tx = session.BeginTransaction())\r
1090                 {\r
1091                     var updatedEntities = session.CreateQuery(hqlUpdate)\r
1092                         .SetString("path", absolutePath)\r
1093                         .SetEnum("status", newStatus)\r
1094                         .ExecuteUpdate();\r
1095                     if (updatedEntities == 0)\r
1096                     {\r
1097                         var newState = new FileState\r
1098                         {\r
1099                             FilePath = absolutePath,                            \r
1100                             OverlayStatus = newStatus,\r
1101                             ETag = Signature.MERKLE_EMPTY,\r
1102                             //LastMD5=String.Empty,\r
1103                             IsFolder = Directory.Exists(absolutePath)\r
1104                         };\r
1105                         session.SaveOrUpdate(newState);\r
1106                         session.Flush();\r
1107                     }\r
1108                     tx.Commit();\r
1109                     return null;\r
1110                 }\r
1111             }, null);\r
1112 \r
1113         }\r
1114 \r
1115     }\r
1116 \r
1117    \r
1118 }\r