root / trunk / Pithos.Core / Agents / FileAgent.cs @ db8a9589
History | View | Annotate | Download (19.9 kB)
1 |
#region |
---|---|
2 |
/* ----------------------------------------------------------------------- |
3 |
* <copyright file="FileAgent.cs" company="GRNet"> |
4 |
* |
5 |
* Copyright 2011-2012 GRNET S.A. All rights reserved. |
6 |
* |
7 |
* Redistribution and use in source and binary forms, with or |
8 |
* without modification, are permitted provided that the following |
9 |
* conditions are met: |
10 |
* |
11 |
* 1. Redistributions of source code must retain the above |
12 |
* copyright notice, this list of conditions and the following |
13 |
* disclaimer. |
14 |
* |
15 |
* 2. Redistributions in binary form must reproduce the above |
16 |
* copyright notice, this list of conditions and the following |
17 |
* disclaimer in the documentation and/or other materials |
18 |
* provided with the distribution. |
19 |
* |
20 |
* |
21 |
* THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
22 |
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
23 |
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
24 |
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
25 |
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
26 |
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
27 |
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
28 |
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
29 |
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
30 |
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
31 |
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
32 |
* POSSIBILITY OF SUCH DAMAGE. |
33 |
* |
34 |
* The views and conclusions contained in the software and |
35 |
* documentation are those of the authors and should not be |
36 |
* interpreted as representing official policies, either expressed |
37 |
* or implied, of GRNET S.A. |
38 |
* </copyright> |
39 |
* ----------------------------------------------------------------------- |
40 |
*/ |
41 |
#endregion |
42 |
using System; |
43 |
using System.Collections.Generic; |
44 |
using System.ComponentModel.Composition; |
45 |
using System.Diagnostics; |
46 |
using System.Diagnostics.Contracts; |
47 |
using System.IO; |
48 |
using System.Linq; |
49 |
using System.Reflection; |
50 |
using System.Text; |
51 |
using System.Threading.Tasks; |
52 |
using Pithos.Interfaces; |
53 |
using Pithos.Network; |
54 |
using log4net; |
55 |
using log4net.Core; |
56 |
|
57 |
namespace Pithos.Core.Agents |
58 |
{ |
59 |
// [Export] |
60 |
public class FileAgent |
61 |
{ |
62 |
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); |
63 |
|
64 |
Agent<WorkflowState> _agent; |
65 |
private FileSystemWatcher _watcher; |
66 |
private FileSystemWatcherAdapter _adapter; |
67 |
|
68 |
//[Import] |
69 |
public IStatusKeeper StatusKeeper { get; set; } |
70 |
//[Import] |
71 |
public IPithosWorkflow Workflow { get; set; } |
72 |
//[Import] |
73 |
public WorkflowAgent WorkflowAgent { get; set; } |
74 |
|
75 |
private AccountInfo AccountInfo { get; set; } |
76 |
|
77 |
internal string RootPath { get; set; } |
78 |
|
79 |
|
80 |
public void Start(AccountInfo accountInfo,string rootPath) |
81 |
{ |
82 |
if (accountInfo==null) |
83 |
throw new ArgumentNullException("accountInfo"); |
84 |
if (String.IsNullOrWhiteSpace(rootPath)) |
85 |
throw new ArgumentNullException("rootPath"); |
86 |
if (!Path.IsPathRooted(rootPath)) |
87 |
throw new ArgumentException("rootPath must be an absolute path","rootPath"); |
88 |
Contract.EndContractBlock(); |
89 |
|
90 |
AccountInfo = accountInfo; |
91 |
RootPath = rootPath; |
92 |
_watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true}; |
93 |
_adapter = new FileSystemWatcherAdapter(_watcher); |
94 |
|
95 |
_adapter.Changed += OnFileEvent; |
96 |
_adapter.Created += OnFileEvent; |
97 |
_adapter.Deleted += OnFileEvent; |
98 |
_adapter.Renamed += OnRenameEvent; |
99 |
_adapter.Moved += OnMoveEvent; |
100 |
_watcher.EnableRaisingEvents = true; |
101 |
|
102 |
|
103 |
_agent = Agent<WorkflowState>.Start(inbox => |
104 |
{ |
105 |
Action loop = null; |
106 |
loop = () => |
107 |
{ |
108 |
var message = inbox.Receive(); |
109 |
var process=message.Then(Process,inbox.CancellationToken); |
110 |
inbox.LoopAsync(process,loop,ex=> |
111 |
Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex)); |
112 |
}; |
113 |
loop(); |
114 |
}); |
115 |
} |
116 |
|
117 |
private Task<object> Process(WorkflowState state) |
118 |
{ |
119 |
if (state==null) |
120 |
throw new ArgumentNullException("state"); |
121 |
Contract.EndContractBlock(); |
122 |
|
123 |
if (Ignore(state.Path)) |
124 |
return CompletedTask<object>.Default; |
125 |
|
126 |
var networkState = NetworkGate.GetNetworkState(state.Path); |
127 |
//Skip if the file is already being downloaded or uploaded and |
128 |
//the change is create or modify |
129 |
if (networkState != NetworkOperation.None && |
130 |
( |
131 |
state.TriggeringChange == WatcherChangeTypes.Created || |
132 |
state.TriggeringChange == WatcherChangeTypes.Changed |
133 |
)) |
134 |
return CompletedTask<object>.Default; |
135 |
|
136 |
try |
137 |
{ |
138 |
UpdateFileStatus(state); |
139 |
UpdateOverlayStatus(state); |
140 |
UpdateFileChecksum(state); |
141 |
WorkflowAgent.Post(state); |
142 |
} |
143 |
catch (IOException exc) |
144 |
{ |
145 |
if (File.Exists(state.Path)) |
146 |
{ |
147 |
Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc); |
148 |
_agent.Post(state); |
149 |
} |
150 |
else |
151 |
{ |
152 |
Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc); |
153 |
} |
154 |
} |
155 |
catch (Exception exc) |
156 |
{ |
157 |
Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}", |
158 |
state.Path, exc); |
159 |
} |
160 |
return CompletedTask<object>.Default; |
161 |
} |
162 |
|
163 |
public bool Pause |
164 |
{ |
165 |
get { return _watcher == null || !_watcher.EnableRaisingEvents; } |
166 |
set |
167 |
{ |
168 |
if (_watcher != null) |
169 |
_watcher.EnableRaisingEvents = !value; |
170 |
} |
171 |
} |
172 |
|
173 |
public string CachePath { get; set; } |
174 |
|
175 |
private List<string> _selectivePaths = new List<string>(); |
176 |
public List<string> SelectivePaths |
177 |
{ |
178 |
get { return _selectivePaths; } |
179 |
set { _selectivePaths = value; } |
180 |
} |
181 |
|
182 |
|
183 |
public void Post(WorkflowState workflowState) |
184 |
{ |
185 |
if (workflowState == null) |
186 |
throw new ArgumentNullException("workflowState"); |
187 |
Contract.EndContractBlock(); |
188 |
|
189 |
_agent.Post(workflowState); |
190 |
} |
191 |
|
192 |
public void Stop() |
193 |
{ |
194 |
if (_watcher != null) |
195 |
{ |
196 |
_watcher.Changed -= OnFileEvent; |
197 |
_watcher.Created -= OnFileEvent; |
198 |
_watcher.Deleted -= OnFileEvent; |
199 |
_watcher.Renamed -= OnRenameEvent; |
200 |
_watcher.Dispose(); |
201 |
} |
202 |
_watcher = null; |
203 |
|
204 |
if (_agent!=null) |
205 |
_agent.Stop(); |
206 |
} |
207 |
|
208 |
// Enumerate all files in the Pithos directory except those in the Fragment folder |
209 |
// and files with a .ignore extension |
210 |
public IEnumerable<string> EnumerateFiles(string searchPattern="*") |
211 |
{ |
212 |
var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories) |
213 |
where !Ignore(filePath) |
214 |
select filePath; |
215 |
return monitoredFiles; |
216 |
} |
217 |
|
218 |
public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*") |
219 |
{ |
220 |
var rootDir = new DirectoryInfo(RootPath); |
221 |
var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories) |
222 |
where !Ignore(file.FullName) |
223 |
select file; |
224 |
return monitoredFiles; |
225 |
} |
226 |
|
227 |
public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*") |
228 |
{ |
229 |
var rootDir = new DirectoryInfo(RootPath); |
230 |
var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories) |
231 |
where !Ignore(file.FullName) |
232 |
select file.AsRelativeUrlTo(RootPath); |
233 |
return monitoredFiles; |
234 |
} |
235 |
|
236 |
public IEnumerable<string> EnumerateFilesSystemInfosAsRelativeUrls(string searchPattern="*") |
237 |
{ |
238 |
var rootDir = new DirectoryInfo(RootPath); |
239 |
var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories) |
240 |
where !Ignore(file.FullName) |
241 |
select file.AsRelativeUrlTo(RootPath); |
242 |
return monitoredFiles; |
243 |
} |
244 |
|
245 |
|
246 |
|
247 |
|
248 |
private bool Ignore(string filePath) |
249 |
{ |
250 |
//Ignore all first-level directories and files (ie at the container folders level) |
251 |
if (FoundBelowRoot(filePath, RootPath,1)) |
252 |
return true; |
253 |
|
254 |
//Ignore first-level items under the "others" folder (ie at the accounts folders level). |
255 |
var othersPath = Path.Combine(RootPath, FolderConstants.OthersFolder); |
256 |
if (FoundBelowRoot(filePath, othersPath,1)) |
257 |
return true; |
258 |
|
259 |
//Ignore second-level (container) folders under the "others" folder (ie at the container folders level). |
260 |
if (FoundBelowRoot(filePath, othersPath,2)) |
261 |
return true; |
262 |
|
263 |
|
264 |
//Ignore anything happening in the cache path |
265 |
if (filePath.StartsWith(CachePath)) |
266 |
return true; |
267 |
if (_ignoreFiles.ContainsKey(filePath.ToLower())) |
268 |
return true; |
269 |
|
270 |
//Ignore if selective synchronization is defined, |
271 |
return SelectivePaths.Count > 0 |
272 |
//And the target file is not below any of the selective paths |
273 |
&& !SelectivePaths.Any(filePath.IsAtOrDirectlyBelow); |
274 |
} |
275 |
|
276 |
/* private static bool FoundInRoot(string filePath, string rootPath) |
277 |
{ |
278 |
//var rootDirectory = new DirectoryInfo(rootPath); |
279 |
|
280 |
//If the paths are equal, return true |
281 |
if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase)) |
282 |
return true; |
283 |
|
284 |
//If the filepath is below the root path |
285 |
if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture)) |
286 |
{ |
287 |
//Get the relative path |
288 |
var relativePath = filePath.Substring(rootPath.Length + 1); |
289 |
//If the relativePath does NOT contains a path separator, we found a match |
290 |
return (!relativePath.Contains(@"\")); |
291 |
} |
292 |
|
293 |
//If the filepath is not under the root path, return false |
294 |
return false; |
295 |
}*/ |
296 |
|
297 |
|
298 |
private static bool FoundBelowRoot(string filePath, string rootPath,int level) |
299 |
{ |
300 |
//var rootDirectory = new DirectoryInfo(rootPath); |
301 |
|
302 |
//If the paths are equal, return true |
303 |
if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase)) |
304 |
return true; |
305 |
|
306 |
//If the filepath is below the root path |
307 |
if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture)) |
308 |
{ |
309 |
//Get the relative path |
310 |
var relativePath = filePath.Substring(rootPath.Length + 1); |
311 |
//If the relativePath does NOT contains a path separator, we found a match |
312 |
var levels=relativePath.ToCharArray().Count(c=>c=='\\')+1; |
313 |
return levels==level; |
314 |
} |
315 |
|
316 |
//If the filepath is not under the root path, return false |
317 |
return false; |
318 |
} |
319 |
|
320 |
//Post a Change message for all events except rename |
321 |
void OnFileEvent(object sender, FileSystemEventArgs e) |
322 |
{ |
323 |
//Ignore events that affect the cache folder |
324 |
var filePath = e.FullPath; |
325 |
if (Ignore(filePath)) |
326 |
return; |
327 |
|
328 |
_agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType }); |
329 |
} |
330 |
|
331 |
|
332 |
//Post a Change message for renames containing the old and new names |
333 |
void OnRenameEvent(object sender, RenamedEventArgs e) |
334 |
{ |
335 |
var oldFullPath = e.OldFullPath; |
336 |
var fullPath = e.FullPath; |
337 |
if (Ignore(oldFullPath) || Ignore(fullPath)) |
338 |
return; |
339 |
|
340 |
_agent.Post(new WorkflowState |
341 |
{ |
342 |
AccountInfo=AccountInfo, |
343 |
OldPath = oldFullPath, |
344 |
OldFileName = e.OldName, |
345 |
Path = fullPath, |
346 |
FileName = e.Name, |
347 |
TriggeringChange = e.ChangeType |
348 |
}); |
349 |
} |
350 |
|
351 |
//Post a Change message for renames containing the old and new names |
352 |
void OnMoveEvent(object sender, MovedEventArgs e) |
353 |
{ |
354 |
var oldFullPath = e.OldFullPath; |
355 |
var fullPath = e.FullPath; |
356 |
if (Ignore(oldFullPath) || Ignore(fullPath)) |
357 |
return; |
358 |
|
359 |
_agent.Post(new WorkflowState |
360 |
{ |
361 |
AccountInfo=AccountInfo, |
362 |
OldPath = oldFullPath, |
363 |
OldFileName = e.OldName, |
364 |
Path = fullPath, |
365 |
FileName = e.Name, |
366 |
TriggeringChange = e.ChangeType |
367 |
}); |
368 |
} |
369 |
|
370 |
|
371 |
|
372 |
private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus> |
373 |
{ |
374 |
{WatcherChangeTypes.Created,FileStatus.Created}, |
375 |
{WatcherChangeTypes.Changed,FileStatus.Modified}, |
376 |
{WatcherChangeTypes.Deleted,FileStatus.Deleted}, |
377 |
{WatcherChangeTypes.Renamed,FileStatus.Renamed} |
378 |
}; |
379 |
|
380 |
private Dictionary<string, string> _ignoreFiles=new Dictionary<string, string>(); |
381 |
|
382 |
private WorkflowState UpdateFileStatus(WorkflowState state) |
383 |
{ |
384 |
if (state==null) |
385 |
throw new ArgumentNullException("state"); |
386 |
if (String.IsNullOrWhiteSpace(state.Path)) |
387 |
throw new ArgumentException("The state's Path can't be empty","state"); |
388 |
Contract.EndContractBlock(); |
389 |
|
390 |
var path = state.Path; |
391 |
var status = _statusDict[state.TriggeringChange]; |
392 |
var oldStatus = Workflow.StatusKeeper.GetFileStatus(path); |
393 |
if (status == oldStatus) |
394 |
{ |
395 |
state.Status = status; |
396 |
state.Skip = true; |
397 |
return state; |
398 |
} |
399 |
if (state.Status == FileStatus.Renamed) |
400 |
Workflow.ClearFileStatus(path); |
401 |
|
402 |
state.Status = Workflow.SetFileStatus(path, status); |
403 |
return state; |
404 |
} |
405 |
|
406 |
private WorkflowState UpdateOverlayStatus(WorkflowState state) |
407 |
{ |
408 |
if (state==null) |
409 |
throw new ArgumentNullException("state"); |
410 |
Contract.EndContractBlock(); |
411 |
|
412 |
if (state.Skip) |
413 |
return state; |
414 |
|
415 |
switch (state.Status) |
416 |
{ |
417 |
case FileStatus.Created: |
418 |
case FileStatus.Modified: |
419 |
this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified); |
420 |
break; |
421 |
case FileStatus.Deleted: |
422 |
//this.StatusAgent.RemoveFileOverlayStatus(state.Path); |
423 |
break; |
424 |
case FileStatus.Renamed: |
425 |
this.StatusKeeper.ClearFileStatus(state.OldPath); |
426 |
this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified); |
427 |
break; |
428 |
case FileStatus.Unchanged: |
429 |
this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal); |
430 |
break; |
431 |
} |
432 |
|
433 |
if (state.Status == FileStatus.Deleted) |
434 |
NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path)); |
435 |
else |
436 |
NativeMethods.RaiseChangeNotification(state.Path); |
437 |
return state; |
438 |
} |
439 |
|
440 |
|
441 |
private WorkflowState UpdateFileChecksum(WorkflowState state) |
442 |
{ |
443 |
if (state.Skip) |
444 |
return state; |
445 |
|
446 |
if (state.Status == FileStatus.Deleted) |
447 |
return state; |
448 |
|
449 |
var path = state.Path; |
450 |
//Skip calculation for folders |
451 |
if (Directory.Exists(path)) |
452 |
return state; |
453 |
|
454 |
var info = new FileInfo(path); |
455 |
string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash); |
456 |
StatusKeeper.UpdateFileChecksum(path, hash); |
457 |
|
458 |
state.Hash = hash; |
459 |
return state; |
460 |
} |
461 |
|
462 |
//Does the file exist in the container's local folder? |
463 |
public bool Exists(string relativePath) |
464 |
{ |
465 |
if (String.IsNullOrWhiteSpace(relativePath)) |
466 |
throw new ArgumentNullException("relativePath"); |
467 |
//A RootPath must be set before calling this method |
468 |
if (String.IsNullOrWhiteSpace(RootPath)) |
469 |
throw new InvalidOperationException("RootPath was not set"); |
470 |
Contract.EndContractBlock(); |
471 |
//Create the absolute path by combining the RootPath with the relativePath |
472 |
var absolutePath=Path.Combine(RootPath, relativePath); |
473 |
//Is this a valid file? |
474 |
if (File.Exists(absolutePath)) |
475 |
return true; |
476 |
//Or a directory? |
477 |
if (Directory.Exists(absolutePath)) |
478 |
return true; |
479 |
//Fail if it is neither |
480 |
return false; |
481 |
} |
482 |
|
483 |
public static FileAgent GetFileAgent(AccountInfo accountInfo) |
484 |
{ |
485 |
return GetFileAgent(accountInfo.AccountPath); |
486 |
} |
487 |
|
488 |
public static FileAgent GetFileAgent(string rootPath) |
489 |
{ |
490 |
return AgentLocator<FileAgent>.Get(rootPath.ToLower()); |
491 |
} |
492 |
|
493 |
|
494 |
public FileSystemInfo GetFileSystemInfo(string relativePath) |
495 |
{ |
496 |
if (String.IsNullOrWhiteSpace(relativePath)) |
497 |
throw new ArgumentNullException("relativePath"); |
498 |
//A RootPath must be set before calling this method |
499 |
if (String.IsNullOrWhiteSpace(RootPath)) |
500 |
throw new InvalidOperationException("RootPath was not set"); |
501 |
Contract.EndContractBlock(); |
502 |
|
503 |
var absolutePath = Path.Combine(RootPath, relativePath); |
504 |
|
505 |
if (Directory.Exists(absolutePath)) |
506 |
return new DirectoryInfo(absolutePath).WithProperCapitalization(); |
507 |
else |
508 |
return new FileInfo(absolutePath).WithProperCapitalization(); |
509 |
|
510 |
} |
511 |
|
512 |
public void Delete(string relativePath) |
513 |
{ |
514 |
var absolutePath = Path.Combine(RootPath, relativePath).ToLower(); |
515 |
if (File.Exists(absolutePath)) |
516 |
{ |
517 |
try |
518 |
{ |
519 |
File.Delete(absolutePath); |
520 |
} |
521 |
//The file may have been deleted by another thread. Just ignore the relevant exception |
522 |
catch (FileNotFoundException) { } |
523 |
} |
524 |
else if (Directory.Exists(absolutePath)) |
525 |
{ |
526 |
try |
527 |
{ |
528 |
Directory.Delete(absolutePath, true); |
529 |
} |
530 |
//The directory may have been deleted by another thread. Just ignore the relevant exception |
531 |
catch (DirectoryNotFoundException){} |
532 |
} |
533 |
|
534 |
//_ignoreFiles[absolutePath] = absolutePath; |
535 |
StatusKeeper.ClearFileStatus(absolutePath); |
536 |
} |
537 |
} |
538 |
} |