Statistics
| Branch: | Revision:

root / trunk / Pithos.Core / Agents / StatusAgent.cs @ 5e31048f

History | View | Annotate | Download (21 kB)

1
using System;
2
using System.Collections.Generic;
3
using System.ComponentModel.Composition;
4
using System.Diagnostics;
5
using System.Diagnostics.Contracts;
6
using System.IO;
7
using System.Linq;
8
using System.Threading;
9
using System.Threading.Tasks;
10
using System.Threading.Tasks.Dataflow;
11
using Castle.ActiveRecord;
12
using Castle.ActiveRecord.Framework.Config;
13
using Pithos.Interfaces;
14
using Pithos.Network;
15
using log4net;
16
using log4net.Appender;
17
using log4net.Config;
18
using log4net.Layout;
19

    
20
namespace Pithos.Core.Agents
21
{
22
    [Export(typeof(IStatusChecker)),Export(typeof(IStatusKeeper))]
23
    public class StatusAgent:IStatusChecker,IStatusKeeper
24
    {
25
        [System.ComponentModel.Composition.Import]
26
        public IPithosSettings Settings { get; set; }
27

    
28
        //private Agent<Action> _persistenceAgent;
29
        private ActionBlock<Action> _persistenceAgent;
30

    
31

    
32
        private static readonly ILog Log = LogManager.GetLogger("StatusAgent");
33

    
34
        public StatusAgent()
35
        {            
36
            var appDataPath = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
37
            _pithosDataPath = Path.Combine(appDataPath , "Pithos");
38

    
39
            if (!Directory.Exists(_pithosDataPath))
40
                Directory.CreateDirectory(_pithosDataPath);
41
            var source = GetConfiguration(_pithosDataPath);
42
            ActiveRecordStarter.Initialize(source,typeof(FileState),typeof(FileTag));
43
            ActiveRecordStarter.UpdateSchema();
44

    
45
            if (!File.Exists(Path.Combine(_pithosDataPath ,"pithos.db")))
46
                ActiveRecordStarter.CreateSchema();
47
        }        
48

    
49
        private static InPlaceConfigurationSource GetConfiguration(string pithosDbPath)
50
        {
51
            if (String.IsNullOrWhiteSpace(pithosDbPath))
52
                throw new ArgumentNullException("pithosDbPath");
53
            if (!Path.IsPathRooted(pithosDbPath))
54
                throw new ArgumentException("path must be a rooted path", "pithosDbPath");
55
            Contract.EndContractBlock();
56

    
57
            var properties = new Dictionary<string, string>
58
                                 {
59
                                     {"connection.driver_class", "NHibernate.Driver.SQLite20Driver"},
60
                                     {"dialect", "NHibernate.Dialect.SQLiteDialect"},
61
                                     {"connection.provider", "NHibernate.Connection.DriverConnectionProvider"},
62
                                     {
63
                                         "proxyfactory.factory_class",
64
                                         "NHibernate.ByteCode.Castle.ProxyFactoryFactory, NHibernate.ByteCode.Castle"
65
                                         },
66
                                 };
67

    
68
            var connectionString = String.Format(@"Data Source={0}\pithos.db;Version=3", pithosDbPath);
69
            properties.Add("connection.connection_string", connectionString);
70

    
71
            var source = new InPlaceConfigurationSource();                        
72
            source.Add(typeof (ActiveRecordBase), properties);
73
            source.SetDebugFlag(false);            
74
            return source;
75
        }
76

    
77
        public void StartProcessing(CancellationToken token)
78
        {
79
            _persistenceAgent = new ActionBlock<Action>(async action=>
80
            {
81
                    try
82
                    {
83
                        await TaskEx.Run(action);
84
                    }
85
                    catch (Exception ex)
86
                    {
87
                        Log.ErrorFormat("[ERROR] STATE \n{0}",ex);
88
                    }
89
            });
90
            
91
        }
92

    
93
       
94

    
95
        public void Stop()
96
        {
97
            _persistenceAgent.Complete();            
98
        }
99
       
100

    
101
        public void ProcessExistingFiles(IEnumerable<FileInfo> existingFiles)
102
        {
103
            if(existingFiles  ==null)
104
                throw new ArgumentNullException("existingFiles");
105
            Contract.EndContractBlock();
106
            
107
            //Find new or matching files with a left join to the stored states
108
            var fileStates = FileState.Queryable;
109
            var currentFiles=from file in existingFiles
110
                      join state in fileStates on file.FullName.ToLower() equals state.FilePath.ToLower() into gs
111
                      from substate in gs.DefaultIfEmpty()
112
                               select new {File = file, State = substate};
113

    
114
            //To get the deleted files we must get the states that have no corresponding
115
            //files. 
116
            //We can't use the File.Exists method inside a query, so we get all file paths from the states
117
            var statePaths = (from state in fileStates
118
                             select new {state.Id, state.FilePath}).ToList();
119
            //and check each one
120
            var missingStates= (from path in statePaths
121
                               where !File.Exists(path.FilePath)
122
                               select path.Id).ToList();
123
            //Finally, retrieve the states that correspond to the deleted files            
124
            var deletedFiles = from state in fileStates 
125
                        where missingStates.Contains(state.Id)
126
                        select new { File = default(FileInfo), State = state };
127

    
128
            var pairs = currentFiles.Union(deletedFiles);
129

    
130
            Parallel.ForEach(pairs, pair =>
131
            {
132
                var fileState = pair.State;
133
                var file = pair.File;
134
                if (fileState == null)
135
                {
136
                    //This is a new file
137
                    var fullPath = pair.File.FullName.ToLower();
138
                    var createState = FileState.CreateForAsync(fullPath, this.BlockSize, this.BlockHash);
139
                    createState.ContinueWith(state => _persistenceAgent.Post(state.Result.Create));
140
                }                
141
                else if (file == null)
142
                {
143
                    //This file was deleted while we were down. We should mark it as deleted
144
                    //We have to go through UpdateStatus here because the state object we are using
145
                    //was created by a different ORM session.
146
                    UpdateStatus(fileState.Id,state=> state.FileStatus = FileStatus.Deleted);                    
147
                }
148
                else
149
                {
150
                    //This file has a matching state. Need to check for possible changes
151
                    var hashString = file.CalculateHash(BlockSize,BlockHash);
152
                    //If the hashes don't match the file was changed
153
                    if (fileState.Checksum != hashString)
154
                    {
155
                        UpdateStatus(fileState.Id, state => state.FileStatus = FileStatus.Modified);
156
                    }                    
157
                }
158
            });            
159
         
160
        }
161

    
162
       
163

    
164

    
165
        public string BlockHash { get; set; }
166

    
167
        public int BlockSize { get; set; }
168
        public void ChangeRoots(string oldPath, string newPath)
169
        {
170
            if (String.IsNullOrWhiteSpace(oldPath))
171
                throw new ArgumentNullException("oldPath");
172
            if (!Path.IsPathRooted(oldPath))
173
                throw new ArgumentException("oldPath must be an absolute path", "oldPath");
174
            if (string.IsNullOrWhiteSpace(newPath))
175
                throw new ArgumentNullException("newPath");
176
            if (!Path.IsPathRooted(newPath))
177
                throw new ArgumentException("newPath must be an absolute path", "newPath");
178
            Contract.EndContractBlock();
179

    
180
            FileState.ChangeRootPath(oldPath,newPath);
181

    
182
        }
183

    
184
        private PithosStatus _pithosStatus=PithosStatus.InSynch;       
185

    
186
        public void SetPithosStatus(PithosStatus status)
187
        {
188
            _pithosStatus = status;
189
        }
190

    
191
        public PithosStatus GetPithosStatus()
192
        {
193
            return _pithosStatus;
194
        }
195

    
196

    
197
        private string _pithosDataPath;
198

    
199
        public T GetStatus<T>(string path,Func<FileState,T> getter,T defaultValue )
200
        {
201
            if (String.IsNullOrWhiteSpace(path))
202
                throw new ArgumentNullException("path");
203
            if (!Path.IsPathRooted(path))
204
                throw new ArgumentException("path must be a rooted path", "path");
205
            if (getter == null)
206
                throw new ArgumentNullException("getter");
207
            Contract.EndContractBlock();
208

    
209

    
210
            try
211
            {                
212
                var state = FileState.FindByFilePath(path);
213
                return state == null ? defaultValue : getter(state);
214
            }
215
            catch (Exception exc)
216
            {
217
                Log.ErrorFormat(exc.ToString());
218
                return defaultValue;
219
            }
220
        }
221

    
222
        /// <summary>
223
        /// Sets the status of a file, creating a new FileState entry if one doesn't already exist.
224
        /// </summary>
225
        /// <param name="path"></param>
226
        /// <param name="setter"></param>
227
        public void SetStatus(string path,Action<FileState> setter)
228
        {
229
            if (String.IsNullOrWhiteSpace(path))
230
                throw new ArgumentNullException("path", "path can't be empty");
231
            if (setter==null)
232
                throw new ArgumentNullException("setter", "setter can't be empty");
233
            Contract.EndContractBlock();
234

    
235
            _persistenceAgent.Post(() =>
236
            {
237
                using (new SessionScope())
238
                {
239
                    var filePath = path.ToLower();
240
                    var state = FileState.FindByFilePath(filePath);
241
                    if (state != null)
242
                    {
243
                        setter(state);
244
                        state.Save();
245
                    }
246
                    else
247
                    {
248
                        state = new FileState {FilePath = filePath};
249
                        setter(state);
250
                        state.Save();
251
                    }                    
252
                }
253
            });
254
        }
255

    
256
        /// <summary>
257
        /// Sets the status of a file only if the file already exists
258
        /// </summary>
259
        /// <param name="path"></param>
260
        /// <param name="setter"></param>
261
        private void UpdateStatus(string path, Action<FileState> setter)
262
        {
263
            if (String.IsNullOrWhiteSpace(path))
264
                throw new ArgumentNullException("path");
265
            if (!Path.IsPathRooted(path))
266
                throw new ArgumentException("The path must be rooted", "path");
267
            if (setter == null)
268
                throw new ArgumentNullException("setter");
269
            Contract.EndContractBlock();
270

    
271
            Debug.Assert(!path.Contains(FolderConstants.CacheFolder));
272
            Debug.Assert(!path.EndsWith(".ignore"));
273

    
274
            if (String.IsNullOrWhiteSpace(path))
275
                throw new ArgumentNullException("path", "path can't be empty");
276

    
277
            if (setter == null)
278
                throw new ArgumentNullException("setter", "setter can't be empty");
279

    
280
            _persistenceAgent.Post(() =>
281
            {
282
                using (new SessionScope())
283
                {
284
                    var filePath = path.ToLower();
285

    
286
                    var state = FileState.FindByFilePath(filePath);
287
                    if (state == null)
288
                    {
289
                        Log.WarnFormat("[NOFILE] Unable to set status for {0}.", filePath);
290
                        return;
291
                    }
292
                    setter(state);
293
                    state.Save();
294
                }
295
                
296
            });
297
        }
298
        
299
        /// <summary>
300
        /// Sets the status of a specific state
301
        /// </summary>
302
        /// <param name="path"></param>
303
        /// <param name="setter"></param>
304
        private void UpdateStatus(Guid stateID, Action<FileState> setter)
305
        {
306
            if (setter == null)
307
                throw new ArgumentNullException("setter");
308
            Contract.EndContractBlock();
309

    
310

    
311
            _persistenceAgent.Post(() =>
312
            {
313
                using (new SessionScope())
314
                {
315
                    var state = FileState.Find(stateID);
316
                    if (state == null)
317
                    {
318
                        Log.WarnFormat("[NOFILE] Unable to set status for {0}.", stateID);
319
                        return;
320
                    }
321
                    setter(state);
322
                    state.Save();
323
                }
324
                
325
            });
326
        }
327

    
328
        public FileOverlayStatus GetFileOverlayStatus(string path)
329
        {
330
            if (String.IsNullOrWhiteSpace(path))
331
                throw new ArgumentNullException("path");
332
            if (!Path.IsPathRooted(path))
333
                throw new ArgumentException("The path must be rooted", "path");
334
            Contract.EndContractBlock();
335

    
336
            try
337
            {
338
                var state = FileState.FindByFilePath(path);
339
                return state == null ? FileOverlayStatus.Unversioned : state.OverlayStatus;
340
            }
341
            catch (Exception exc)
342
            {
343
                Log.ErrorFormat(exc.ToString());
344
                return FileOverlayStatus.Unversioned;
345
            }
346
        }
347

    
348
        public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)
349
        {
350
            if (String.IsNullOrWhiteSpace(path))
351
                throw new ArgumentNullException("path");
352
            if (!Path.IsPathRooted(path))
353
                throw new ArgumentException("The path must be rooted","path");
354
            Contract.EndContractBlock();
355

    
356
            SetStatus(path.ToLower(),s=>s.OverlayStatus=overlayStatus);
357
        }
358

    
359
        /*public void RemoveFileOverlayStatus(string path)
360
        {
361
            if (String.IsNullOrWhiteSpace(path))
362
                throw new ArgumentNullException("path");
363
            if (!Path.IsPathRooted(path))
364
                throw new ArgumentException("The path must be rooted", "path");
365
            Contract.EndContractBlock();
366

    
367
            _persistenceAgent.Post(() =>
368
                InnerRemoveFileOverlayStatus(path));
369
        }
370

    
371
        private static void InnerRemoveFileOverlayStatus(string path)
372
        {
373
            if (String.IsNullOrWhiteSpace(path))
374
                throw new ArgumentNullException("path");
375
            if (!Path.IsPathRooted(path))
376
                throw new ArgumentException("The path must be rooted", "path");
377
            Contract.EndContractBlock();
378

    
379
            FileState.DeleteByFilePath(path);            
380
        }*/
381

    
382
        public void RenameFileOverlayStatus(string oldPath, string newPath)
383
        {
384
            if (String.IsNullOrWhiteSpace(oldPath))
385
                throw new ArgumentNullException("oldPath");
386
            if (!Path.IsPathRooted(oldPath))
387
                throw new ArgumentException("The oldPath must be rooted", "oldPath");
388
            if (String.IsNullOrWhiteSpace(newPath))
389
                throw new ArgumentNullException("newPath");
390
            if (!Path.IsPathRooted(newPath))
391
                throw new ArgumentException("The newPath must be rooted", "newPath");
392
            Contract.EndContractBlock();
393

    
394
            _persistenceAgent.Post(() =>
395
                InnerRenameFileOverlayStatus(oldPath, newPath));
396
        }
397

    
398
        private static void InnerRenameFileOverlayStatus(string oldPath, string newPath)
399
        {
400
            if (String.IsNullOrWhiteSpace(oldPath))
401
                throw new ArgumentNullException("oldPath");
402
            if (!Path.IsPathRooted(oldPath))
403
                throw new ArgumentException("The oldPath must be rooted", "oldPath");
404
            if (String.IsNullOrWhiteSpace(newPath))
405
                throw new ArgumentNullException("newPath");
406
            if (!Path.IsPathRooted(newPath))
407
                throw new ArgumentException("The newPath must be rooted", "newPath");
408
            Contract.EndContractBlock();
409

    
410
            var state = FileState.FindByFilePath(oldPath);
411

    
412
            if (state == null)
413
            {
414
                Log.WarnFormat("[NOFILE] Unable to set status for {0}.", oldPath);
415
                return;
416
            }
417
            //NOTE: This will cause problems if path is used as a key in relationships
418
            state.FilePath = newPath;
419
            state.Update();
420
        }
421

    
422
        public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus)
423
        {
424
            if (String.IsNullOrWhiteSpace(path))
425
                throw new ArgumentNullException("path");
426
            if (!Path.IsPathRooted(path))
427
                throw new ArgumentException("The path must be rooted", "path");
428
            Contract.EndContractBlock();
429

    
430
            UpdateStatus(path.ToLower(),state=>
431
                                  {
432
                                      state.FileStatus = fileStatus;
433
                                      state.OverlayStatus = overlayStatus;
434
                                  });            
435
        }
436

    
437
        public void StoreInfo(string path,ObjectInfo objectInfo)
438
        {
439
            if (String.IsNullOrWhiteSpace(path))
440
                throw new ArgumentNullException("path");
441
            if (!Path.IsPathRooted(path))
442
                throw new ArgumentException("The path must be rooted", "path");            
443
            if (objectInfo == null)
444
                throw new ArgumentNullException("objectInfo", "objectInfo can't be empty");
445
            Contract.EndContractBlock();
446

    
447
            _persistenceAgent.Post(() =>
448
            {
449
                var filePath = path.ToLower();
450
                //Load the existing files state and set its properties in one session            
451
                using (new SessionScope())
452
                {
453
                    //Forgetting to use a sessionscope results in two sessions being created, one by 
454
                    //FirstOrDefault and one by Save()
455
                    var state =FileState.FindByFilePath(filePath);
456
                    //Create a new empty state object if this is a new file
457
                    state = state ?? new FileState();
458

    
459
                    state.FilePath = filePath;
460
                    state.Checksum = objectInfo.Hash;
461
                    state.Version = objectInfo.Version;
462
                    state.VersionTimeStamp = objectInfo.VersionTimestamp;
463

    
464
                    state.FileStatus = FileStatus.Unchanged;
465
                    state.OverlayStatus = FileOverlayStatus.Normal;
466
                    
467
                    //Create a list of tags from the ObjectInfo's tag dictionary
468
                    //Make sure to bind each tag to its parent state so we don't have to save each tag separately
469
                    //state.Tags = (from pair in objectInfo.Tags
470
                    //                select
471
                    //                    new FileTag
472
                    //                        {
473
                    //                            FileState = state,
474
                    //                            Name = pair.Key,
475
                    //                            Value = pair.Value
476
                    //                        }
477
                    //            ).ToList();
478

    
479
                    //Do the save
480
                    state.Save();
481
                }
482
            });
483

    
484
        }
485

    
486
        
487
        public void SetFileStatus(string path, FileStatus status)
488
        {            
489
            UpdateStatus(path.ToLower(), state=>state.FileStatus = status);
490
        }
491

    
492
        public FileStatus GetFileStatus(string path)
493
        {
494
            if (String.IsNullOrWhiteSpace(path))
495
                throw new ArgumentNullException("path");
496
            if (!Path.IsPathRooted(path))
497
                throw new ArgumentException("The path must be rooted", "path");
498
            Contract.EndContractBlock();
499

    
500
            var state = FileState.FindByFilePath(path);
501
            return (state==null)?FileStatus.Missing:state.FileStatus ;
502
        }
503

    
504
        public void ClearFileStatus(string path)
505
        {
506
            if (String.IsNullOrWhiteSpace(path))
507
                throw new ArgumentNullException("path");
508
            if (!Path.IsPathRooted(path))
509
                throw new ArgumentException("The path must be rooted", "path");
510
            Contract.EndContractBlock();
511

    
512
            //TODO:SHOULDN'T need both clear file status and remove overlay status
513
            _persistenceAgent.Post(() =>
514
            {
515
                using (new SessionScope())
516
                {
517
                    FileState.DeleteByFilePath(path);
518
                }
519
            });   
520
        }
521

    
522
        public void UpdateFileChecksum(string path, string checksum)
523
        {
524
            if (String.IsNullOrWhiteSpace(path))
525
                throw new ArgumentNullException("path");
526
            if (!Path.IsPathRooted(path))
527
                throw new ArgumentException("The path must be rooted", "path");            
528
            Contract.EndContractBlock();
529

    
530
            _persistenceAgent.Post(() =>
531
            {
532
                using (new SessionScope())
533
                {
534
                    var state = FileState.FindByFilePath(path);
535
                    if (state == null)
536
                    {
537
                        Log.WarnFormat("[NOFILE] Unable to set checkesum for {0}.", path);
538
                        return;
539
                    }
540
                    state.Checksum = checksum;
541
                    state.Update();
542
                }
543
            });
544
        }
545

    
546
    }
547

    
548
   
549
}