Statistics
| Branch: | Revision:

root / trunk / Pithos.Core / Agents / FileAgent.cs @ b666b39a

History | View | Annotate | Download (19.8 kB)

1
#region
2
/* -----------------------------------------------------------------------
3
 * <copyright file="FileAgent.cs" company="GRNet">
4
 * 
5
 * Copyright 2011-2012 GRNET S.A. All rights reserved.
6
 *
7
 * Redistribution and use in source and binary forms, with or
8
 * without modification, are permitted provided that the following
9
 * conditions are met:
10
 *
11
 *   1. Redistributions of source code must retain the above
12
 *      copyright notice, this list of conditions and the following
13
 *      disclaimer.
14
 *
15
 *   2. Redistributions in binary form must reproduce the above
16
 *      copyright notice, this list of conditions and the following
17
 *      disclaimer in the documentation and/or other materials
18
 *      provided with the distribution.
19
 *
20
 *
21
 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32
 * POSSIBILITY OF SUCH DAMAGE.
33
 *
34
 * The views and conclusions contained in the software and
35
 * documentation are those of the authors and should not be
36
 * interpreted as representing official policies, either expressed
37
 * or implied, of GRNET S.A.
38
 * </copyright>
39
 * -----------------------------------------------------------------------
40
 */
41
#endregion
42
using System;
43
using System.Collections.Generic;
44
using System.ComponentModel.Composition;
45
using System.Diagnostics;
46
using System.Diagnostics.Contracts;
47
using System.IO;
48
using System.Linq;
49
using System.Text;
50
using System.Threading.Tasks;
51
using Pithos.Interfaces;
52
using Pithos.Network;
53
using log4net;
54
using log4net.Core;
55

    
56
namespace Pithos.Core.Agents
57
{
58
//    [Export]
59
    public class FileAgent
60
    {
61
        Agent<WorkflowState> _agent;
62
        private FileSystemWatcher _watcher;
63
        private FileSystemWatcherAdapter _adapter;
64

    
65
        //[Import]
66
        public IStatusKeeper StatusKeeper { get; set; }
67
        //[Import]
68
        public IPithosWorkflow Workflow { get; set; }
69
        //[Import]
70
        public WorkflowAgent WorkflowAgent { get; set; }
71

    
72
        private AccountInfo AccountInfo { get; set; }
73

    
74
        internal string RootPath { get;  set; }
75

    
76
        private static readonly ILog Log = LogManager.GetLogger("FileAgent");
77

    
78
        public void Start(AccountInfo accountInfo,string rootPath)
79
        {
80
            if (accountInfo==null)
81
                throw new ArgumentNullException("accountInfo");
82
            if (String.IsNullOrWhiteSpace(rootPath))
83
                throw new ArgumentNullException("rootPath");
84
            if (!Path.IsPathRooted(rootPath))
85
                throw new ArgumentException("rootPath must be an absolute path","rootPath");
86
            Contract.EndContractBlock();
87

    
88
            AccountInfo = accountInfo;
89
            RootPath = rootPath;
90
            _watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true};
91
            _adapter = new FileSystemWatcherAdapter(_watcher);
92

    
93
            _adapter.Changed += OnFileEvent;
94
            _adapter.Created += OnFileEvent;
95
            _adapter.Deleted += OnFileEvent;
96
            _adapter.Renamed += OnRenameEvent;
97
            _adapter.Moved += OnMoveEvent;
98
            _watcher.EnableRaisingEvents = true;
99

    
100

    
101
            _agent = Agent<WorkflowState>.Start(inbox =>
102
            {
103
                Action loop = null;
104
                loop = () =>
105
                {
106
                    var message = inbox.Receive();
107
                    var process=message.Then(Process,inbox.CancellationToken);                    
108
                    inbox.LoopAsync(process,loop,ex=>
109
                        Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
110
                };
111
                loop();
112
            });
113
        }
114

    
115
        private Task<object> Process(WorkflowState state)
116
        {
117
            if (state==null)
118
                throw new ArgumentNullException("state");
119
            Contract.EndContractBlock();
120

    
121
            if (Ignore(state.Path))
122
                return CompletedTask<object>.Default;
123

    
124
            var networkState = NetworkGate.GetNetworkState(state.Path);
125
            //Skip if the file is already being downloaded or uploaded and 
126
            //the change is create or modify
127
            if (networkState != NetworkOperation.None &&
128
                (
129
                    state.TriggeringChange == WatcherChangeTypes.Created ||
130
                    state.TriggeringChange == WatcherChangeTypes.Changed
131
                ))
132
                return CompletedTask<object>.Default;
133

    
134
            try
135
            {
136
                UpdateFileStatus(state);
137
                UpdateOverlayStatus(state);
138
                UpdateFileChecksum(state);
139
                WorkflowAgent.Post(state);
140
            }
141
            catch (IOException exc)
142
            {
143
                if (File.Exists(state.Path))
144
                {
145
                    Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
146
                    _agent.Post(state);
147
                }
148
                else
149
                {
150
                    Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
151
                }
152
            }
153
            catch (Exception exc)
154
            {
155
                Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
156
                               state.Path, exc);
157
            }
158
            return CompletedTask<object>.Default;
159
        }
160

    
161
        public bool Pause
162
        {
163
            get { return _watcher == null || !_watcher.EnableRaisingEvents; }
164
            set
165
            {
166
                if (_watcher != null)
167
                    _watcher.EnableRaisingEvents = !value;                
168
            }
169
        }
170

    
171
        public string CachePath { get; set; }
172

    
173
        private List<string> _selectivePaths = new List<string>();
174
        public List<string> SelectivePaths
175
        {
176
            get { return _selectivePaths; }
177
            set { _selectivePaths = value; }
178
        }
179

    
180

    
181
        public void Post(WorkflowState workflowState)
182
        {
183
            if (workflowState == null)
184
                throw new ArgumentNullException("workflowState");
185
            Contract.EndContractBlock();
186

    
187
            _agent.Post(workflowState);
188
        }
189

    
190
        public void Stop()
191
        {
192
            if (_watcher != null)
193
            {
194
                _watcher.Changed -= OnFileEvent;
195
                _watcher.Created -= OnFileEvent;
196
                _watcher.Deleted -= OnFileEvent;
197
                _watcher.Renamed -= OnRenameEvent;
198
                _watcher.Dispose();
199
            }
200
            _watcher = null;
201

    
202
            if (_agent!=null)
203
                _agent.Stop();
204
        }
205

    
206
        // Enumerate all files in the Pithos directory except those in the Fragment folder
207
        // and files with a .ignore extension
208
        public IEnumerable<string> EnumerateFiles(string searchPattern="*")
209
        {
210
            var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
211
                                 where !Ignore(filePath)
212
                                 select filePath;
213
            return monitoredFiles;
214
        }
215

    
216
        public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
217
        {
218
            var rootDir = new DirectoryInfo(RootPath);
219
            var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
220
                                 where !Ignore(file.FullName)
221
                                 select file;
222
            return monitoredFiles;
223
        }                
224

    
225
        public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
226
        {
227
            var rootDir = new DirectoryInfo(RootPath);
228
            var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
229
                                 where !Ignore(file.FullName)
230
                                 select file.AsRelativeUrlTo(RootPath);
231
            return monitoredFiles;
232
        }                
233

    
234
        public IEnumerable<string> EnumerateFilesSystemInfosAsRelativeUrls(string searchPattern="*")
235
        {
236
            var rootDir = new DirectoryInfo(RootPath);
237
            var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories)
238
                                 where !Ignore(file.FullName)
239
                                 select file.AsRelativeUrlTo(RootPath);
240
            return monitoredFiles;
241
        }                
242

    
243

    
244
        
245

    
246
        private bool Ignore(string filePath)
247
        {
248
            //Ignore all first-level directories and files (ie at the container folders level)
249
            if (FoundBelowRoot(filePath, RootPath,1))
250
                return true;
251

    
252
            //Ignore first-level items under the "others" folder (ie at the accounts folders level).
253
            var othersPath = Path.Combine(RootPath, FolderConstants.OthersFolder);
254
            if (FoundBelowRoot(filePath, othersPath,1))
255
                return true;
256

    
257
            //Ignore second-level (container) folders under the "others" folder (ie at the container folders level). 
258
            if (FoundBelowRoot(filePath, othersPath,2))
259
                return true;            
260

    
261

    
262
            //Ignore anything happening in the cache path
263
            if (filePath.StartsWith(CachePath))
264
                return true;
265
            if (_ignoreFiles.ContainsKey(filePath.ToLower()))
266
                return true;
267

    
268
            //Ignore if selective synchronization is defined, 
269
            return SelectivePaths.Count > 0 
270
                //And the target file is not below any of the selective paths
271
                && !SelectivePaths.Any(filePath.IsAtOrDirectlyBelow);
272
        }
273

    
274
/*        private static bool FoundInRoot(string filePath, string rootPath)
275
        {
276
            //var rootDirectory = new DirectoryInfo(rootPath);
277

    
278
            //If the paths are equal, return true
279
            if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
280
                return true;
281

    
282
            //If the filepath is below the root path
283
            if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
284
            {
285
                //Get the relative path
286
                var relativePath = filePath.Substring(rootPath.Length + 1);
287
                //If the relativePath does NOT contains a path separator, we found a match
288
                return (!relativePath.Contains(@"\"));
289
            }
290

    
291
            //If the filepath is not under the root path, return false
292
            return false;            
293
        }*/
294

    
295

    
296
        private static bool FoundBelowRoot(string filePath, string rootPath,int level)
297
        {
298
            //var rootDirectory = new DirectoryInfo(rootPath);
299

    
300
            //If the paths are equal, return true
301
            if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
302
                return true;
303

    
304
            //If the filepath is below the root path
305
            if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
306
            {
307
                //Get the relative path
308
                var relativePath = filePath.Substring(rootPath.Length + 1);
309
                //If the relativePath does NOT contains a path separator, we found a match
310
                var levels=relativePath.ToCharArray().Count(c=>c=='\\')+1;                
311
                return levels==level;
312
            }
313

    
314
            //If the filepath is not under the root path, return false
315
            return false;            
316
        }
317

    
318
        //Post a Change message for all events except rename
319
        void OnFileEvent(object sender, FileSystemEventArgs e)
320
        {
321
            //Ignore events that affect the cache folder
322
            var filePath = e.FullPath;
323
            if (Ignore(filePath)) 
324
                return;
325

    
326
            _agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
327
        }
328

    
329

    
330
        //Post a Change message for renames containing the old and new names
331
        void OnRenameEvent(object sender, RenamedEventArgs e)
332
        {
333
            var oldFullPath = e.OldFullPath;
334
            var fullPath = e.FullPath;
335
            if (Ignore(oldFullPath) || Ignore(fullPath))
336
                return;
337

    
338
            _agent.Post(new WorkflowState
339
            {
340
                AccountInfo=AccountInfo,
341
                OldPath = oldFullPath,
342
                OldFileName = e.OldName,
343
                Path = fullPath,
344
                FileName = e.Name,
345
                TriggeringChange = e.ChangeType
346
            });
347
        }
348

    
349
        //Post a Change message for renames containing the old and new names
350
        void OnMoveEvent(object sender, MovedEventArgs e)
351
        {
352
            var oldFullPath = e.OldFullPath;
353
            var fullPath = e.FullPath;
354
            if (Ignore(oldFullPath) || Ignore(fullPath))
355
                return;
356

    
357
            _agent.Post(new WorkflowState
358
            {
359
                AccountInfo=AccountInfo,
360
                OldPath = oldFullPath,
361
                OldFileName = e.OldName,
362
                Path = fullPath,
363
                FileName = e.Name,
364
                TriggeringChange = e.ChangeType
365
            });
366
        }
367

    
368

    
369

    
370
        private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
371
                                                                             {
372
            {WatcherChangeTypes.Created,FileStatus.Created},
373
            {WatcherChangeTypes.Changed,FileStatus.Modified},
374
            {WatcherChangeTypes.Deleted,FileStatus.Deleted},
375
            {WatcherChangeTypes.Renamed,FileStatus.Renamed}
376
        };
377

    
378
        private Dictionary<string, string> _ignoreFiles=new Dictionary<string, string>();
379

    
380
        private WorkflowState UpdateFileStatus(WorkflowState state)
381
        {
382
            if (state==null)
383
                throw new ArgumentNullException("state");
384
            if (String.IsNullOrWhiteSpace(state.Path))
385
                throw new ArgumentException("The state's Path can't be empty","state");
386
            Contract.EndContractBlock();
387

    
388
            var path = state.Path;
389
            var status = _statusDict[state.TriggeringChange];
390
            var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
391
            if (status == oldStatus)
392
            {
393
                state.Status = status;
394
                state.Skip = true;
395
                return state;
396
            }
397
            if (state.Status == FileStatus.Renamed)
398
                Workflow.ClearFileStatus(path);
399

    
400
            state.Status = Workflow.SetFileStatus(path, status);
401
            return state;
402
        }
403

    
404
        private WorkflowState UpdateOverlayStatus(WorkflowState state)
405
        {
406
            if (state==null)
407
                throw new ArgumentNullException("state");
408
            Contract.EndContractBlock();
409

    
410
            if (state.Skip)
411
                return state;
412

    
413
            switch (state.Status)
414
            {
415
                case FileStatus.Created:
416
                case FileStatus.Modified:
417
                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
418
                    break;
419
                case FileStatus.Deleted:
420
                    //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
421
                    break;
422
                case FileStatus.Renamed:
423
                    this.StatusKeeper.ClearFileStatus(state.OldPath);
424
                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
425
                    break;
426
                case FileStatus.Unchanged:
427
                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
428
                    break;
429
            }
430

    
431
            if (state.Status == FileStatus.Deleted)
432
                NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
433
            else
434
                NativeMethods.RaiseChangeNotification(state.Path);
435
            return state;
436
        }
437

    
438

    
439
        private WorkflowState UpdateFileChecksum(WorkflowState state)
440
        {
441
            if (state.Skip)
442
                return state;
443

    
444
            if (state.Status == FileStatus.Deleted)
445
                return state;
446

    
447
            var path = state.Path;
448
            //Skip calculation for folders
449
            if (Directory.Exists(path))
450
                return state;
451

    
452
            var info = new FileInfo(path);
453
            string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
454
            StatusKeeper.UpdateFileChecksum(path, hash);
455

    
456
            state.Hash = hash;
457
            return state;
458
        }
459

    
460
        //Does the file exist in the container's local folder?
461
        public bool Exists(string relativePath)
462
        {
463
            if (String.IsNullOrWhiteSpace(relativePath))
464
                throw new ArgumentNullException("relativePath");
465
            //A RootPath must be set before calling this method
466
            if (String.IsNullOrWhiteSpace(RootPath))
467
                throw new InvalidOperationException("RootPath was not set");
468
            Contract.EndContractBlock();
469
            //Create the absolute path by combining the RootPath with the relativePath
470
            var absolutePath=Path.Combine(RootPath, relativePath);
471
            //Is this a valid file?
472
            if (File.Exists(absolutePath))
473
                return true;
474
            //Or a directory?
475
            if (Directory.Exists(absolutePath))
476
                return true;
477
            //Fail if it is neither
478
            return false;
479
        }
480

    
481
        public static FileAgent GetFileAgent(AccountInfo accountInfo)
482
        {
483
            return GetFileAgent(accountInfo.AccountPath);
484
        }
485

    
486
        public static FileAgent GetFileAgent(string rootPath)
487
        {
488
            return AgentLocator<FileAgent>.Get(rootPath.ToLower());
489
        }
490

    
491

    
492
        public FileSystemInfo GetFileSystemInfo(string relativePath)
493
        {
494
            if (String.IsNullOrWhiteSpace(relativePath))
495
                throw new ArgumentNullException("relativePath");
496
            //A RootPath must be set before calling this method
497
            if (String.IsNullOrWhiteSpace(RootPath))
498
                throw new InvalidOperationException("RootPath was not set");            
499
            Contract.EndContractBlock();            
500

    
501
            var absolutePath = Path.Combine(RootPath, relativePath);
502

    
503
            if (Directory.Exists(absolutePath))
504
                return new DirectoryInfo(absolutePath).WithProperCapitalization();
505
            else
506
                return new FileInfo(absolutePath).WithProperCapitalization();
507
            
508
        }
509

    
510
        public void Delete(string relativePath)
511
        {
512
            var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
513
            if (File.Exists(absolutePath))
514
            {    
515
                try
516
                {
517
                    File.Delete(absolutePath);
518
                }
519
                //The file may have been deleted by another thread. Just ignore the relevant exception
520
                catch (FileNotFoundException) { }
521
            }
522
            else if (Directory.Exists(absolutePath))
523
            {
524
                try
525
                {
526
                    Directory.Delete(absolutePath, true);
527
                }
528
                //The directory may have been deleted by another thread. Just ignore the relevant exception
529
                catch (DirectoryNotFoundException){}                
530
            }
531
        
532
            //_ignoreFiles[absolutePath] = absolutePath;                
533
            StatusKeeper.ClearFileStatus(absolutePath);
534
        }
535
    }
536
}