Statistics
| Branch: | Revision:

root / trunk / Pithos.Core / Agents / FileAgent.cs @ db8a9589

History | View | Annotate | Download (19.9 kB)

1
#region
2
/* -----------------------------------------------------------------------
3
 * <copyright file="FileAgent.cs" company="GRNet">
4
 * 
5
 * Copyright 2011-2012 GRNET S.A. All rights reserved.
6
 *
7
 * Redistribution and use in source and binary forms, with or
8
 * without modification, are permitted provided that the following
9
 * conditions are met:
10
 *
11
 *   1. Redistributions of source code must retain the above
12
 *      copyright notice, this list of conditions and the following
13
 *      disclaimer.
14
 *
15
 *   2. Redistributions in binary form must reproduce the above
16
 *      copyright notice, this list of conditions and the following
17
 *      disclaimer in the documentation and/or other materials
18
 *      provided with the distribution.
19
 *
20
 *
21
 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32
 * POSSIBILITY OF SUCH DAMAGE.
33
 *
34
 * The views and conclusions contained in the software and
35
 * documentation are those of the authors and should not be
36
 * interpreted as representing official policies, either expressed
37
 * or implied, of GRNET S.A.
38
 * </copyright>
39
 * -----------------------------------------------------------------------
40
 */
41
#endregion
42
using System;
43
using System.Collections.Generic;
44
using System.ComponentModel.Composition;
45
using System.Diagnostics;
46
using System.Diagnostics.Contracts;
47
using System.IO;
48
using System.Linq;
49
using System.Reflection;
50
using System.Text;
51
using System.Threading.Tasks;
52
using Pithos.Interfaces;
53
using Pithos.Network;
54
using log4net;
55
using log4net.Core;
56

    
57
namespace Pithos.Core.Agents
58
{
59
//    [Export]
60
    public class FileAgent
61
    {
62
        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
63

    
64
        Agent<WorkflowState> _agent;
65
        private FileSystemWatcher _watcher;
66
        private FileSystemWatcherAdapter _adapter;
67

    
68
        //[Import]
69
        public IStatusKeeper StatusKeeper { get; set; }
70
        //[Import]
71
        public IPithosWorkflow Workflow { get; set; }
72
        //[Import]
73
        public WorkflowAgent WorkflowAgent { get; set; }
74

    
75
        private AccountInfo AccountInfo { get; set; }
76

    
77
        internal string RootPath { get;  set; }
78

    
79

    
80
        public void Start(AccountInfo accountInfo,string rootPath)
81
        {
82
            if (accountInfo==null)
83
                throw new ArgumentNullException("accountInfo");
84
            if (String.IsNullOrWhiteSpace(rootPath))
85
                throw new ArgumentNullException("rootPath");
86
            if (!Path.IsPathRooted(rootPath))
87
                throw new ArgumentException("rootPath must be an absolute path","rootPath");
88
            Contract.EndContractBlock();
89

    
90
            AccountInfo = accountInfo;
91
            RootPath = rootPath;
92
            _watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true};
93
            _adapter = new FileSystemWatcherAdapter(_watcher);
94

    
95
            _adapter.Changed += OnFileEvent;
96
            _adapter.Created += OnFileEvent;
97
            _adapter.Deleted += OnFileEvent;
98
            _adapter.Renamed += OnRenameEvent;
99
            _adapter.Moved += OnMoveEvent;
100
            _watcher.EnableRaisingEvents = true;
101

    
102

    
103
            _agent = Agent<WorkflowState>.Start(inbox =>
104
            {
105
                Action loop = null;
106
                loop = () =>
107
                {
108
                    var message = inbox.Receive();
109
                    var process=message.Then(Process,inbox.CancellationToken);                    
110
                    inbox.LoopAsync(process,loop,ex=>
111
                        Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
112
                };
113
                loop();
114
            });
115
        }
116

    
117
        private Task<object> Process(WorkflowState state)
118
        {
119
            if (state==null)
120
                throw new ArgumentNullException("state");
121
            Contract.EndContractBlock();
122

    
123
            if (Ignore(state.Path))
124
                return CompletedTask<object>.Default;
125

    
126
            var networkState = NetworkGate.GetNetworkState(state.Path);
127
            //Skip if the file is already being downloaded or uploaded and 
128
            //the change is create or modify
129
            if (networkState != NetworkOperation.None &&
130
                (
131
                    state.TriggeringChange == WatcherChangeTypes.Created ||
132
                    state.TriggeringChange == WatcherChangeTypes.Changed
133
                ))
134
                return CompletedTask<object>.Default;
135

    
136
            try
137
            {
138
                UpdateFileStatus(state);
139
                UpdateOverlayStatus(state);
140
                UpdateFileChecksum(state);
141
                WorkflowAgent.Post(state);
142
            }
143
            catch (IOException exc)
144
            {
145
                if (File.Exists(state.Path))
146
                {
147
                    Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
148
                    _agent.Post(state);
149
                }
150
                else
151
                {
152
                    Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
153
                }
154
            }
155
            catch (Exception exc)
156
            {
157
                Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
158
                               state.Path, exc);
159
            }
160
            return CompletedTask<object>.Default;
161
        }
162

    
163
        public bool Pause
164
        {
165
            get { return _watcher == null || !_watcher.EnableRaisingEvents; }
166
            set
167
            {
168
                if (_watcher != null)
169
                    _watcher.EnableRaisingEvents = !value;                
170
            }
171
        }
172

    
173
        public string CachePath { get; set; }
174

    
175
        private List<string> _selectivePaths = new List<string>();
176
        public List<string> SelectivePaths
177
        {
178
            get { return _selectivePaths; }
179
            set { _selectivePaths = value; }
180
        }
181

    
182

    
183
        public void Post(WorkflowState workflowState)
184
        {
185
            if (workflowState == null)
186
                throw new ArgumentNullException("workflowState");
187
            Contract.EndContractBlock();
188

    
189
            _agent.Post(workflowState);
190
        }
191

    
192
        public void Stop()
193
        {
194
            if (_watcher != null)
195
            {
196
                _watcher.Changed -= OnFileEvent;
197
                _watcher.Created -= OnFileEvent;
198
                _watcher.Deleted -= OnFileEvent;
199
                _watcher.Renamed -= OnRenameEvent;
200
                _watcher.Dispose();
201
            }
202
            _watcher = null;
203

    
204
            if (_agent!=null)
205
                _agent.Stop();
206
        }
207

    
208
        // Enumerate all files in the Pithos directory except those in the Fragment folder
209
        // and files with a .ignore extension
210
        public IEnumerable<string> EnumerateFiles(string searchPattern="*")
211
        {
212
            var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
213
                                 where !Ignore(filePath)
214
                                 select filePath;
215
            return monitoredFiles;
216
        }
217

    
218
        public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
219
        {
220
            var rootDir = new DirectoryInfo(RootPath);
221
            var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
222
                                 where !Ignore(file.FullName)
223
                                 select file;
224
            return monitoredFiles;
225
        }                
226

    
227
        public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
228
        {
229
            var rootDir = new DirectoryInfo(RootPath);
230
            var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
231
                                 where !Ignore(file.FullName)
232
                                 select file.AsRelativeUrlTo(RootPath);
233
            return monitoredFiles;
234
        }                
235

    
236
        public IEnumerable<string> EnumerateFilesSystemInfosAsRelativeUrls(string searchPattern="*")
237
        {
238
            var rootDir = new DirectoryInfo(RootPath);
239
            var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories)
240
                                 where !Ignore(file.FullName)
241
                                 select file.AsRelativeUrlTo(RootPath);
242
            return monitoredFiles;
243
        }                
244

    
245

    
246
        
247

    
248
        private bool Ignore(string filePath)
249
        {
250
            //Ignore all first-level directories and files (ie at the container folders level)
251
            if (FoundBelowRoot(filePath, RootPath,1))
252
                return true;
253

    
254
            //Ignore first-level items under the "others" folder (ie at the accounts folders level).
255
            var othersPath = Path.Combine(RootPath, FolderConstants.OthersFolder);
256
            if (FoundBelowRoot(filePath, othersPath,1))
257
                return true;
258

    
259
            //Ignore second-level (container) folders under the "others" folder (ie at the container folders level). 
260
            if (FoundBelowRoot(filePath, othersPath,2))
261
                return true;            
262

    
263

    
264
            //Ignore anything happening in the cache path
265
            if (filePath.StartsWith(CachePath))
266
                return true;
267
            if (_ignoreFiles.ContainsKey(filePath.ToLower()))
268
                return true;
269

    
270
            //Ignore if selective synchronization is defined, 
271
            return SelectivePaths.Count > 0 
272
                //And the target file is not below any of the selective paths
273
                && !SelectivePaths.Any(filePath.IsAtOrDirectlyBelow);
274
        }
275

    
276
/*        private static bool FoundInRoot(string filePath, string rootPath)
277
        {
278
            //var rootDirectory = new DirectoryInfo(rootPath);
279

    
280
            //If the paths are equal, return true
281
            if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
282
                return true;
283

    
284
            //If the filepath is below the root path
285
            if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
286
            {
287
                //Get the relative path
288
                var relativePath = filePath.Substring(rootPath.Length + 1);
289
                //If the relativePath does NOT contains a path separator, we found a match
290
                return (!relativePath.Contains(@"\"));
291
            }
292

    
293
            //If the filepath is not under the root path, return false
294
            return false;            
295
        }*/
296

    
297

    
298
        private static bool FoundBelowRoot(string filePath, string rootPath,int level)
299
        {
300
            //var rootDirectory = new DirectoryInfo(rootPath);
301

    
302
            //If the paths are equal, return true
303
            if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
304
                return true;
305

    
306
            //If the filepath is below the root path
307
            if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
308
            {
309
                //Get the relative path
310
                var relativePath = filePath.Substring(rootPath.Length + 1);
311
                //If the relativePath does NOT contains a path separator, we found a match
312
                var levels=relativePath.ToCharArray().Count(c=>c=='\\')+1;                
313
                return levels==level;
314
            }
315

    
316
            //If the filepath is not under the root path, return false
317
            return false;            
318
        }
319

    
320
        //Post a Change message for all events except rename
321
        void OnFileEvent(object sender, FileSystemEventArgs e)
322
        {
323
            //Ignore events that affect the cache folder
324
            var filePath = e.FullPath;
325
            if (Ignore(filePath)) 
326
                return;
327

    
328
            _agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
329
        }
330

    
331

    
332
        //Post a Change message for renames containing the old and new names
333
        void OnRenameEvent(object sender, RenamedEventArgs e)
334
        {
335
            var oldFullPath = e.OldFullPath;
336
            var fullPath = e.FullPath;
337
            if (Ignore(oldFullPath) || Ignore(fullPath))
338
                return;
339

    
340
            _agent.Post(new WorkflowState
341
            {
342
                AccountInfo=AccountInfo,
343
                OldPath = oldFullPath,
344
                OldFileName = e.OldName,
345
                Path = fullPath,
346
                FileName = e.Name,
347
                TriggeringChange = e.ChangeType
348
            });
349
        }
350

    
351
        //Post a Change message for renames containing the old and new names
352
        void OnMoveEvent(object sender, MovedEventArgs e)
353
        {
354
            var oldFullPath = e.OldFullPath;
355
            var fullPath = e.FullPath;
356
            if (Ignore(oldFullPath) || Ignore(fullPath))
357
                return;
358

    
359
            _agent.Post(new WorkflowState
360
            {
361
                AccountInfo=AccountInfo,
362
                OldPath = oldFullPath,
363
                OldFileName = e.OldName,
364
                Path = fullPath,
365
                FileName = e.Name,
366
                TriggeringChange = e.ChangeType
367
            });
368
        }
369

    
370

    
371

    
372
        private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
373
                                                                             {
374
            {WatcherChangeTypes.Created,FileStatus.Created},
375
            {WatcherChangeTypes.Changed,FileStatus.Modified},
376
            {WatcherChangeTypes.Deleted,FileStatus.Deleted},
377
            {WatcherChangeTypes.Renamed,FileStatus.Renamed}
378
        };
379

    
380
        private Dictionary<string, string> _ignoreFiles=new Dictionary<string, string>();
381

    
382
        private WorkflowState UpdateFileStatus(WorkflowState state)
383
        {
384
            if (state==null)
385
                throw new ArgumentNullException("state");
386
            if (String.IsNullOrWhiteSpace(state.Path))
387
                throw new ArgumentException("The state's Path can't be empty","state");
388
            Contract.EndContractBlock();
389

    
390
            var path = state.Path;
391
            var status = _statusDict[state.TriggeringChange];
392
            var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
393
            if (status == oldStatus)
394
            {
395
                state.Status = status;
396
                state.Skip = true;
397
                return state;
398
            }
399
            if (state.Status == FileStatus.Renamed)
400
                Workflow.ClearFileStatus(path);
401

    
402
            state.Status = Workflow.SetFileStatus(path, status);
403
            return state;
404
        }
405

    
406
        private WorkflowState UpdateOverlayStatus(WorkflowState state)
407
        {
408
            if (state==null)
409
                throw new ArgumentNullException("state");
410
            Contract.EndContractBlock();
411

    
412
            if (state.Skip)
413
                return state;
414

    
415
            switch (state.Status)
416
            {
417
                case FileStatus.Created:
418
                case FileStatus.Modified:
419
                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
420
                    break;
421
                case FileStatus.Deleted:
422
                    //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
423
                    break;
424
                case FileStatus.Renamed:
425
                    this.StatusKeeper.ClearFileStatus(state.OldPath);
426
                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
427
                    break;
428
                case FileStatus.Unchanged:
429
                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
430
                    break;
431
            }
432

    
433
            if (state.Status == FileStatus.Deleted)
434
                NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
435
            else
436
                NativeMethods.RaiseChangeNotification(state.Path);
437
            return state;
438
        }
439

    
440

    
441
        private WorkflowState UpdateFileChecksum(WorkflowState state)
442
        {
443
            if (state.Skip)
444
                return state;
445

    
446
            if (state.Status == FileStatus.Deleted)
447
                return state;
448

    
449
            var path = state.Path;
450
            //Skip calculation for folders
451
            if (Directory.Exists(path))
452
                return state;
453

    
454
            var info = new FileInfo(path);
455
            string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
456
            StatusKeeper.UpdateFileChecksum(path, hash);
457

    
458
            state.Hash = hash;
459
            return state;
460
        }
461

    
462
        //Does the file exist in the container's local folder?
463
        public bool Exists(string relativePath)
464
        {
465
            if (String.IsNullOrWhiteSpace(relativePath))
466
                throw new ArgumentNullException("relativePath");
467
            //A RootPath must be set before calling this method
468
            if (String.IsNullOrWhiteSpace(RootPath))
469
                throw new InvalidOperationException("RootPath was not set");
470
            Contract.EndContractBlock();
471
            //Create the absolute path by combining the RootPath with the relativePath
472
            var absolutePath=Path.Combine(RootPath, relativePath);
473
            //Is this a valid file?
474
            if (File.Exists(absolutePath))
475
                return true;
476
            //Or a directory?
477
            if (Directory.Exists(absolutePath))
478
                return true;
479
            //Fail if it is neither
480
            return false;
481
        }
482

    
483
        public static FileAgent GetFileAgent(AccountInfo accountInfo)
484
        {
485
            return GetFileAgent(accountInfo.AccountPath);
486
        }
487

    
488
        public static FileAgent GetFileAgent(string rootPath)
489
        {
490
            return AgentLocator<FileAgent>.Get(rootPath.ToLower());
491
        }
492

    
493

    
494
        public FileSystemInfo GetFileSystemInfo(string relativePath)
495
        {
496
            if (String.IsNullOrWhiteSpace(relativePath))
497
                throw new ArgumentNullException("relativePath");
498
            //A RootPath must be set before calling this method
499
            if (String.IsNullOrWhiteSpace(RootPath))
500
                throw new InvalidOperationException("RootPath was not set");            
501
            Contract.EndContractBlock();            
502

    
503
            var absolutePath = Path.Combine(RootPath, relativePath);
504

    
505
            if (Directory.Exists(absolutePath))
506
                return new DirectoryInfo(absolutePath).WithProperCapitalization();
507
            else
508
                return new FileInfo(absolutePath).WithProperCapitalization();
509
            
510
        }
511

    
512
        public void Delete(string relativePath)
513
        {
514
            var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
515
            if (File.Exists(absolutePath))
516
            {    
517
                try
518
                {
519
                    File.Delete(absolutePath);
520
                }
521
                //The file may have been deleted by another thread. Just ignore the relevant exception
522
                catch (FileNotFoundException) { }
523
            }
524
            else if (Directory.Exists(absolutePath))
525
            {
526
                try
527
                {
528
                    Directory.Delete(absolutePath, true);
529
                }
530
                //The directory may have been deleted by another thread. Just ignore the relevant exception
531
                catch (DirectoryNotFoundException){}                
532
            }
533
        
534
            //_ignoreFiles[absolutePath] = absolutePath;                
535
            StatusKeeper.ClearFileStatus(absolutePath);
536
        }
537
    }
538
}