Statistics
| Branch: | Revision:

root / trunk / Pithos.Core / Agents / FileAgent.cs @ 78ebfd2d

History | View | Annotate | Download (14.6 kB)

1
´╗┐using System;
2
using System.Collections.Generic;
3
using System.ComponentModel.Composition;
4
using System.Diagnostics;
5
using System.Diagnostics.Contracts;
6
using System.IO;
7
using System.Linq;
8
using System.Text;
9
using System.Threading.Tasks;
10
using Pithos.Interfaces;
11
using Pithos.Network;
12
using log4net;
13
using log4net.Core;
14

    
15
namespace Pithos.Core.Agents
16
{
17
//    [Export]
18
    public class FileAgent
19
    {
20
        Agent<WorkflowState> _agent;
21
        private FileSystemWatcher _watcher;
22
        private FileSystemWatcherAdapter _adapter;
23

    
24
        //[Import]
25
        public IStatusKeeper StatusKeeper { get; set; }
26
        //[Import]
27
        public IPithosWorkflow Workflow { get; set; }
28
        //[Import]
29
        public WorkflowAgent WorkflowAgent { get; set; }
30

    
31
        private AccountInfo AccountInfo { get; set; }
32

    
33
        private string RootPath { get;  set; }
34

    
35
        private static readonly ILog Log = LogManager.GetLogger("FileAgent");
36

    
37
        public void Start(AccountInfo accountInfo,string rootPath)
38
        {
39
            if (accountInfo==null)
40
                throw new ArgumentNullException("accountInfo");
41
            if (String.IsNullOrWhiteSpace(rootPath))
42
                throw new ArgumentNullException("rootPath");
43
            if (!Path.IsPathRooted(rootPath))
44
                throw new ArgumentException("rootPath must be an absolute path","rootPath");
45
            Contract.EndContractBlock();
46

    
47
            AccountInfo = accountInfo;
48
            RootPath = rootPath;
49
            _watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true};
50
            _adapter = new FileSystemWatcherAdapter(_watcher);
51

    
52
            _adapter.Changed += OnFileEvent;
53
            _adapter.Created += OnFileEvent;
54
            _adapter.Deleted += OnFileEvent;
55
            _adapter.Renamed += OnRenameEvent;
56
            _adapter.Moved += OnMoveEvent;
57
            _watcher.EnableRaisingEvents = true;
58

    
59

    
60
            _agent = Agent<WorkflowState>.Start(inbox =>
61
            {
62
                Action loop = null;
63
                loop = () =>
64
                {
65
                    var message = inbox.Receive();
66
                    var process=message.Then(Process,inbox.CancellationToken);
67

    
68
                    inbox.LoopAsync(process,loop,ex=>
69
                        Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
70
                };
71
                loop();
72
            });
73
        }
74

    
75
        private Task<object> Process(WorkflowState state)
76
        {
77
            if (state==null)
78
                throw new ArgumentNullException("state");
79
            Contract.EndContractBlock();
80

    
81
            if (Ignore(state.Path))
82
                return CompletedTask<object>.Default;
83

    
84
            var networkState = NetworkGate.GetNetworkState(state.Path);
85
            //Skip if the file is already being downloaded or uploaded and 
86
            //the change is create or modify
87
            if (networkState != NetworkOperation.None &&
88
                (
89
                    state.TriggeringChange == WatcherChangeTypes.Created ||
90
                    state.TriggeringChange == WatcherChangeTypes.Changed
91
                ))
92
                return CompletedTask<object>.Default;
93

    
94
            try
95
            {
96
                UpdateFileStatus(state);
97
                UpdateOverlayStatus(state);
98
                UpdateFileChecksum(state);
99
                WorkflowAgent.Post(state);
100
            }
101
            catch (IOException exc)
102
            {
103
                if (File.Exists(state.Path))
104
                {
105
                    Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
106
                    _agent.Post(state);
107
                }
108
                else
109
                {
110
                    Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
111
                }
112
            }
113
            catch (Exception exc)
114
            {
115
                Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
116
                               state.Path, exc);
117
            }
118
            return CompletedTask<object>.Default;
119
        }
120

    
121
        public bool Pause
122
        {
123
            get { return _watcher == null || !_watcher.EnableRaisingEvents; }
124
            set
125
            {
126
                if (_watcher != null)
127
                    _watcher.EnableRaisingEvents = !value;                
128
            }
129
        }
130

    
131
        public string CachePath { get; set; }
132

    
133
        private List<string> _selectivePaths = new List<string>();
134
        public List<string> SelectivePaths
135
        {
136
            get { return _selectivePaths; }
137
            set { _selectivePaths = value; }
138
        }
139

    
140

    
141
        public void Post(WorkflowState workflowState)
142
        {
143
            if (workflowState == null)
144
                throw new ArgumentNullException("workflowState");
145
            Contract.EndContractBlock();
146

    
147
            _agent.Post(workflowState);
148
        }
149

    
150
        public void Stop()
151
        {
152
            if (_watcher != null)
153
            {
154
                _watcher.Changed -= OnFileEvent;
155
                _watcher.Created -= OnFileEvent;
156
                _watcher.Deleted -= OnFileEvent;
157
                _watcher.Renamed -= OnRenameEvent;
158
                _watcher.Dispose();
159
            }
160
            _watcher = null;
161

    
162
            if (_agent!=null)
163
                _agent.Stop();
164
        }
165

    
166
        // Enumerate all files in the Pithos directory except those in the Fragment folder
167
        // and files with a .ignore extension
168
        public IEnumerable<string> EnumerateFiles(string searchPattern="*")
169
        {
170
            var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
171
                                 where !Ignore(filePath)
172
                                 select filePath;
173
            return monitoredFiles;
174
        }
175

    
176
        public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
177
        {
178
            var rootDir = new DirectoryInfo(RootPath);
179
            var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
180
                                 where !Ignore(file.FullName)
181
                                 select file;
182
            return monitoredFiles;
183
        }                
184

    
185
        public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
186
        {
187
            var rootDir = new DirectoryInfo(RootPath);
188
            var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
189
                                 where !Ignore(file.FullName)
190
                                 select file.AsRelativeUrlTo(RootPath);
191
            return monitoredFiles;
192
        }                
193

    
194

    
195
        
196

    
197
        private bool Ignore(string filePath)
198
        {
199
            var pithosPath = Path.Combine(RootPath, "pithos");
200
            if (pithosPath.Equals(filePath, StringComparison.InvariantCultureIgnoreCase))
201
                return true;
202
            if (filePath.StartsWith(CachePath))
203
                return true;
204
            if (_ignoreFiles.ContainsKey(filePath.ToLower()))
205
                return true;
206
            return false;
207
        }
208

    
209
        //Post a Change message for all events except rename
210
        void OnFileEvent(object sender, FileSystemEventArgs e)
211
        {
212
            //Ignore events that affect the cache folder
213
            var filePath = e.FullPath;
214
            if (Ignore(filePath)) 
215
                return;
216

    
217
            _agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
218
        }
219

    
220

    
221
        //Post a Change message for renames containing the old and new names
222
        void OnRenameEvent(object sender, RenamedEventArgs e)
223
        {
224
            var oldFullPath = e.OldFullPath;
225
            var fullPath = e.FullPath;
226
            if (Ignore(oldFullPath) || Ignore(fullPath))
227
                return;
228

    
229
            _agent.Post(new WorkflowState
230
            {
231
                AccountInfo=AccountInfo,
232
                OldPath = oldFullPath,
233
                OldFileName = e.OldName,
234
                Path = fullPath,
235
                FileName = e.Name,
236
                TriggeringChange = e.ChangeType
237
            });
238
        }
239

    
240
        //Post a Change message for renames containing the old and new names
241
        void OnMoveEvent(object sender, MovedEventArgs e)
242
        {
243
            var oldFullPath = e.OldFullPath;
244
            var fullPath = e.FullPath;
245
            if (Ignore(oldFullPath) || Ignore(fullPath))
246
                return;
247

    
248
            _agent.Post(new WorkflowState
249
            {
250
                AccountInfo=AccountInfo,
251
                OldPath = oldFullPath,
252
                OldFileName = e.OldName,
253
                Path = fullPath,
254
                FileName = e.Name,
255
                TriggeringChange = e.ChangeType
256
            });
257
        }
258

    
259

    
260

    
261
        private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
262
        {
263
            {WatcherChangeTypes.Created,FileStatus.Created},
264
            {WatcherChangeTypes.Changed,FileStatus.Modified},
265
            {WatcherChangeTypes.Deleted,FileStatus.Deleted},
266
            {WatcherChangeTypes.Renamed,FileStatus.Renamed}
267
        };
268

    
269
        private Dictionary<string,string> _ignoreFiles=new Dictionary<string, string>();
270

    
271
        private WorkflowState UpdateFileStatus(WorkflowState state)
272
        {
273
            if (state==null)
274
                throw new ArgumentNullException("state");
275
            if (String.IsNullOrWhiteSpace(state.Path))
276
                throw new ArgumentException("The state's Path can't be empty","state");
277
            Contract.EndContractBlock();
278

    
279
            var path = state.Path;
280
            var status = _statusDict[state.TriggeringChange];
281
            var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
282
            if (status == oldStatus)
283
            {
284
                state.Status = status;
285
                state.Skip = true;
286
                return state;
287
            }
288
            if (state.Status == FileStatus.Renamed)
289
                Workflow.ClearFileStatus(path);
290

    
291
            state.Status = Workflow.SetFileStatus(path, status);
292
            return state;
293
        }
294

    
295
        private WorkflowState UpdateOverlayStatus(WorkflowState state)
296
        {
297
            if (state==null)
298
                throw new ArgumentNullException("state");
299
            Contract.EndContractBlock();
300

    
301
            if (state.Skip)
302
                return state;
303

    
304
            switch (state.Status)
305
            {
306
                case FileStatus.Created:
307
                case FileStatus.Modified:
308
                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
309
                    break;
310
                case FileStatus.Deleted:
311
                    //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
312
                    break;
313
                case FileStatus.Renamed:
314
                    this.StatusKeeper.ClearFileStatus(state.OldPath);
315
                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
316
                    break;
317
                case FileStatus.Unchanged:
318
                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
319
                    break;
320
            }
321

    
322
            if (state.Status == FileStatus.Deleted)
323
                NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
324
            else
325
                NativeMethods.RaiseChangeNotification(state.Path);
326
            return state;
327
        }
328

    
329

    
330
        private WorkflowState UpdateFileChecksum(WorkflowState state)
331
        {
332
            if (state.Skip)
333
                return state;
334

    
335
            if (state.Status == FileStatus.Deleted)
336
                return state;
337

    
338
            var path = state.Path;
339
            //Skip calculation for folders
340
            if (Directory.Exists(path))
341
                return state;
342

    
343
            var info = new FileInfo(path);
344
            string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
345
            StatusKeeper.UpdateFileChecksum(path, hash);
346

    
347
            state.Hash = hash;
348
            return state;
349
        }
350

    
351
        //Does the file exist in the container's local folder?
352
        public bool Exists(string relativePath)
353
        {
354
            if (String.IsNullOrWhiteSpace(relativePath))
355
                throw new ArgumentNullException("relativePath");
356
            //A RootPath must be set before calling this method
357
            if (String.IsNullOrWhiteSpace(RootPath))
358
                throw new InvalidOperationException("RootPath was not set");
359
            Contract.EndContractBlock();
360
            //Create the absolute path by combining the RootPath with the relativePath
361
            var absolutePath=Path.Combine(RootPath, relativePath);
362
            //Is this a valid file?
363
            if (File.Exists(absolutePath))
364
                return true;
365
            //Or a directory?
366
            if (Directory.Exists(absolutePath))
367
                return true;
368
            //Fail if it is neither
369
            return false;
370
        }
371

    
372
        public FileSystemInfo GetFileSystemInfo(string relativePath)
373
        {
374
            if (String.IsNullOrWhiteSpace(relativePath))
375
                throw new ArgumentNullException("relativePath");
376
            //A RootPath must be set before calling this method
377
            if (String.IsNullOrWhiteSpace(RootPath))
378
                throw new InvalidOperationException("RootPath was not set");            
379
            Contract.EndContractBlock();            
380

    
381
            var absolutePath = Path.Combine(RootPath, relativePath);
382

    
383
            if (Directory.Exists(absolutePath))
384
                return new DirectoryInfo(absolutePath).WithProperCapitalization();
385
            else
386
                return new FileInfo(absolutePath).WithProperCapitalization();
387
            
388
        }
389

    
390
        public void Delete(string relativePath)
391
        {
392
            var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
393
            if (File.Exists(absolutePath))
394
            {    
395
                try
396
                {
397
                    File.Delete(absolutePath);
398
                }
399
                //The file may have been deleted by another thread. Just ignore the relevant exception
400
                catch (FileNotFoundException) { }
401
            }
402
            else if (Directory.Exists(absolutePath))
403
            {
404
                try
405
                {
406
                    Directory.Delete(absolutePath, true);
407
                }
408
                //The directory may have been deleted by another thread. Just ignore the relevant exception
409
                catch (DirectoryNotFoundException){}                
410
            }
411
        
412
            //_ignoreFiles[absolutePath] = absolutePath;                
413
            StatusKeeper.ClearFileStatus(absolutePath);
414
        }
415
    }
416
}