root / trunk / Pithos.Core / Agents / FileAgent.cs @ 78ebfd2d
History | View | Annotate | Download (14.6 kB)
1 |
using System; |
---|---|
2 |
using System.Collections.Generic; |
3 |
using System.ComponentModel.Composition; |
4 |
using System.Diagnostics; |
5 |
using System.Diagnostics.Contracts; |
6 |
using System.IO; |
7 |
using System.Linq; |
8 |
using System.Text; |
9 |
using System.Threading.Tasks; |
10 |
using Pithos.Interfaces; |
11 |
using Pithos.Network; |
12 |
using log4net; |
13 |
using log4net.Core; |
14 |
|
15 |
namespace Pithos.Core.Agents |
16 |
{ |
17 |
// [Export] |
18 |
public class FileAgent |
19 |
{ |
20 |
Agent<WorkflowState> _agent; |
21 |
private FileSystemWatcher _watcher; |
22 |
private FileSystemWatcherAdapter _adapter; |
23 |
|
24 |
//[Import] |
25 |
public IStatusKeeper StatusKeeper { get; set; } |
26 |
//[Import] |
27 |
public IPithosWorkflow Workflow { get; set; } |
28 |
//[Import] |
29 |
public WorkflowAgent WorkflowAgent { get; set; } |
30 |
|
31 |
private AccountInfo AccountInfo { get; set; } |
32 |
|
33 |
private string RootPath { get; set; } |
34 |
|
35 |
private static readonly ILog Log = LogManager.GetLogger("FileAgent"); |
36 |
|
37 |
public void Start(AccountInfo accountInfo,string rootPath) |
38 |
{ |
39 |
if (accountInfo==null) |
40 |
throw new ArgumentNullException("accountInfo"); |
41 |
if (String.IsNullOrWhiteSpace(rootPath)) |
42 |
throw new ArgumentNullException("rootPath"); |
43 |
if (!Path.IsPathRooted(rootPath)) |
44 |
throw new ArgumentException("rootPath must be an absolute path","rootPath"); |
45 |
Contract.EndContractBlock(); |
46 |
|
47 |
AccountInfo = accountInfo; |
48 |
RootPath = rootPath; |
49 |
_watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true}; |
50 |
_adapter = new FileSystemWatcherAdapter(_watcher); |
51 |
|
52 |
_adapter.Changed += OnFileEvent; |
53 |
_adapter.Created += OnFileEvent; |
54 |
_adapter.Deleted += OnFileEvent; |
55 |
_adapter.Renamed += OnRenameEvent; |
56 |
_adapter.Moved += OnMoveEvent; |
57 |
_watcher.EnableRaisingEvents = true; |
58 |
|
59 |
|
60 |
_agent = Agent<WorkflowState>.Start(inbox => |
61 |
{ |
62 |
Action loop = null; |
63 |
loop = () => |
64 |
{ |
65 |
var message = inbox.Receive(); |
66 |
var process=message.Then(Process,inbox.CancellationToken); |
67 |
|
68 |
inbox.LoopAsync(process,loop,ex=> |
69 |
Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex)); |
70 |
}; |
71 |
loop(); |
72 |
}); |
73 |
} |
74 |
|
75 |
private Task<object> Process(WorkflowState state) |
76 |
{ |
77 |
if (state==null) |
78 |
throw new ArgumentNullException("state"); |
79 |
Contract.EndContractBlock(); |
80 |
|
81 |
if (Ignore(state.Path)) |
82 |
return CompletedTask<object>.Default; |
83 |
|
84 |
var networkState = NetworkGate.GetNetworkState(state.Path); |
85 |
//Skip if the file is already being downloaded or uploaded and |
86 |
//the change is create or modify |
87 |
if (networkState != NetworkOperation.None && |
88 |
( |
89 |
state.TriggeringChange == WatcherChangeTypes.Created || |
90 |
state.TriggeringChange == WatcherChangeTypes.Changed |
91 |
)) |
92 |
return CompletedTask<object>.Default; |
93 |
|
94 |
try |
95 |
{ |
96 |
UpdateFileStatus(state); |
97 |
UpdateOverlayStatus(state); |
98 |
UpdateFileChecksum(state); |
99 |
WorkflowAgent.Post(state); |
100 |
} |
101 |
catch (IOException exc) |
102 |
{ |
103 |
if (File.Exists(state.Path)) |
104 |
{ |
105 |
Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc); |
106 |
_agent.Post(state); |
107 |
} |
108 |
else |
109 |
{ |
110 |
Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc); |
111 |
} |
112 |
} |
113 |
catch (Exception exc) |
114 |
{ |
115 |
Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}", |
116 |
state.Path, exc); |
117 |
} |
118 |
return CompletedTask<object>.Default; |
119 |
} |
120 |
|
121 |
public bool Pause |
122 |
{ |
123 |
get { return _watcher == null || !_watcher.EnableRaisingEvents; } |
124 |
set |
125 |
{ |
126 |
if (_watcher != null) |
127 |
_watcher.EnableRaisingEvents = !value; |
128 |
} |
129 |
} |
130 |
|
131 |
public string CachePath { get; set; } |
132 |
|
133 |
private List<string> _selectivePaths = new List<string>(); |
134 |
public List<string> SelectivePaths |
135 |
{ |
136 |
get { return _selectivePaths; } |
137 |
set { _selectivePaths = value; } |
138 |
} |
139 |
|
140 |
|
141 |
public void Post(WorkflowState workflowState) |
142 |
{ |
143 |
if (workflowState == null) |
144 |
throw new ArgumentNullException("workflowState"); |
145 |
Contract.EndContractBlock(); |
146 |
|
147 |
_agent.Post(workflowState); |
148 |
} |
149 |
|
150 |
public void Stop() |
151 |
{ |
152 |
if (_watcher != null) |
153 |
{ |
154 |
_watcher.Changed -= OnFileEvent; |
155 |
_watcher.Created -= OnFileEvent; |
156 |
_watcher.Deleted -= OnFileEvent; |
157 |
_watcher.Renamed -= OnRenameEvent; |
158 |
_watcher.Dispose(); |
159 |
} |
160 |
_watcher = null; |
161 |
|
162 |
if (_agent!=null) |
163 |
_agent.Stop(); |
164 |
} |
165 |
|
166 |
// Enumerate all files in the Pithos directory except those in the Fragment folder |
167 |
// and files with a .ignore extension |
168 |
public IEnumerable<string> EnumerateFiles(string searchPattern="*") |
169 |
{ |
170 |
var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories) |
171 |
where !Ignore(filePath) |
172 |
select filePath; |
173 |
return monitoredFiles; |
174 |
} |
175 |
|
176 |
public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*") |
177 |
{ |
178 |
var rootDir = new DirectoryInfo(RootPath); |
179 |
var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories) |
180 |
where !Ignore(file.FullName) |
181 |
select file; |
182 |
return monitoredFiles; |
183 |
} |
184 |
|
185 |
public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*") |
186 |
{ |
187 |
var rootDir = new DirectoryInfo(RootPath); |
188 |
var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories) |
189 |
where !Ignore(file.FullName) |
190 |
select file.AsRelativeUrlTo(RootPath); |
191 |
return monitoredFiles; |
192 |
} |
193 |
|
194 |
|
195 |
|
196 |
|
197 |
private bool Ignore(string filePath) |
198 |
{ |
199 |
var pithosPath = Path.Combine(RootPath, "pithos"); |
200 |
if (pithosPath.Equals(filePath, StringComparison.InvariantCultureIgnoreCase)) |
201 |
return true; |
202 |
if (filePath.StartsWith(CachePath)) |
203 |
return true; |
204 |
if (_ignoreFiles.ContainsKey(filePath.ToLower())) |
205 |
return true; |
206 |
return false; |
207 |
} |
208 |
|
209 |
//Post a Change message for all events except rename |
210 |
void OnFileEvent(object sender, FileSystemEventArgs e) |
211 |
{ |
212 |
//Ignore events that affect the cache folder |
213 |
var filePath = e.FullPath; |
214 |
if (Ignore(filePath)) |
215 |
return; |
216 |
|
217 |
_agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType }); |
218 |
} |
219 |
|
220 |
|
221 |
//Post a Change message for renames containing the old and new names |
222 |
void OnRenameEvent(object sender, RenamedEventArgs e) |
223 |
{ |
224 |
var oldFullPath = e.OldFullPath; |
225 |
var fullPath = e.FullPath; |
226 |
if (Ignore(oldFullPath) || Ignore(fullPath)) |
227 |
return; |
228 |
|
229 |
_agent.Post(new WorkflowState |
230 |
{ |
231 |
AccountInfo=AccountInfo, |
232 |
OldPath = oldFullPath, |
233 |
OldFileName = e.OldName, |
234 |
Path = fullPath, |
235 |
FileName = e.Name, |
236 |
TriggeringChange = e.ChangeType |
237 |
}); |
238 |
} |
239 |
|
240 |
//Post a Change message for renames containing the old and new names |
241 |
void OnMoveEvent(object sender, MovedEventArgs e) |
242 |
{ |
243 |
var oldFullPath = e.OldFullPath; |
244 |
var fullPath = e.FullPath; |
245 |
if (Ignore(oldFullPath) || Ignore(fullPath)) |
246 |
return; |
247 |
|
248 |
_agent.Post(new WorkflowState |
249 |
{ |
250 |
AccountInfo=AccountInfo, |
251 |
OldPath = oldFullPath, |
252 |
OldFileName = e.OldName, |
253 |
Path = fullPath, |
254 |
FileName = e.Name, |
255 |
TriggeringChange = e.ChangeType |
256 |
}); |
257 |
} |
258 |
|
259 |
|
260 |
|
261 |
private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus> |
262 |
{ |
263 |
{WatcherChangeTypes.Created,FileStatus.Created}, |
264 |
{WatcherChangeTypes.Changed,FileStatus.Modified}, |
265 |
{WatcherChangeTypes.Deleted,FileStatus.Deleted}, |
266 |
{WatcherChangeTypes.Renamed,FileStatus.Renamed} |
267 |
}; |
268 |
|
269 |
private Dictionary<string,string> _ignoreFiles=new Dictionary<string, string>(); |
270 |
|
271 |
private WorkflowState UpdateFileStatus(WorkflowState state) |
272 |
{ |
273 |
if (state==null) |
274 |
throw new ArgumentNullException("state"); |
275 |
if (String.IsNullOrWhiteSpace(state.Path)) |
276 |
throw new ArgumentException("The state's Path can't be empty","state"); |
277 |
Contract.EndContractBlock(); |
278 |
|
279 |
var path = state.Path; |
280 |
var status = _statusDict[state.TriggeringChange]; |
281 |
var oldStatus = Workflow.StatusKeeper.GetFileStatus(path); |
282 |
if (status == oldStatus) |
283 |
{ |
284 |
state.Status = status; |
285 |
state.Skip = true; |
286 |
return state; |
287 |
} |
288 |
if (state.Status == FileStatus.Renamed) |
289 |
Workflow.ClearFileStatus(path); |
290 |
|
291 |
state.Status = Workflow.SetFileStatus(path, status); |
292 |
return state; |
293 |
} |
294 |
|
295 |
private WorkflowState UpdateOverlayStatus(WorkflowState state) |
296 |
{ |
297 |
if (state==null) |
298 |
throw new ArgumentNullException("state"); |
299 |
Contract.EndContractBlock(); |
300 |
|
301 |
if (state.Skip) |
302 |
return state; |
303 |
|
304 |
switch (state.Status) |
305 |
{ |
306 |
case FileStatus.Created: |
307 |
case FileStatus.Modified: |
308 |
this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified); |
309 |
break; |
310 |
case FileStatus.Deleted: |
311 |
//this.StatusAgent.RemoveFileOverlayStatus(state.Path); |
312 |
break; |
313 |
case FileStatus.Renamed: |
314 |
this.StatusKeeper.ClearFileStatus(state.OldPath); |
315 |
this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified); |
316 |
break; |
317 |
case FileStatus.Unchanged: |
318 |
this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal); |
319 |
break; |
320 |
} |
321 |
|
322 |
if (state.Status == FileStatus.Deleted) |
323 |
NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path)); |
324 |
else |
325 |
NativeMethods.RaiseChangeNotification(state.Path); |
326 |
return state; |
327 |
} |
328 |
|
329 |
|
330 |
private WorkflowState UpdateFileChecksum(WorkflowState state) |
331 |
{ |
332 |
if (state.Skip) |
333 |
return state; |
334 |
|
335 |
if (state.Status == FileStatus.Deleted) |
336 |
return state; |
337 |
|
338 |
var path = state.Path; |
339 |
//Skip calculation for folders |
340 |
if (Directory.Exists(path)) |
341 |
return state; |
342 |
|
343 |
var info = new FileInfo(path); |
344 |
string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash); |
345 |
StatusKeeper.UpdateFileChecksum(path, hash); |
346 |
|
347 |
state.Hash = hash; |
348 |
return state; |
349 |
} |
350 |
|
351 |
//Does the file exist in the container's local folder? |
352 |
public bool Exists(string relativePath) |
353 |
{ |
354 |
if (String.IsNullOrWhiteSpace(relativePath)) |
355 |
throw new ArgumentNullException("relativePath"); |
356 |
//A RootPath must be set before calling this method |
357 |
if (String.IsNullOrWhiteSpace(RootPath)) |
358 |
throw new InvalidOperationException("RootPath was not set"); |
359 |
Contract.EndContractBlock(); |
360 |
//Create the absolute path by combining the RootPath with the relativePath |
361 |
var absolutePath=Path.Combine(RootPath, relativePath); |
362 |
//Is this a valid file? |
363 |
if (File.Exists(absolutePath)) |
364 |
return true; |
365 |
//Or a directory? |
366 |
if (Directory.Exists(absolutePath)) |
367 |
return true; |
368 |
//Fail if it is neither |
369 |
return false; |
370 |
} |
371 |
|
372 |
public FileSystemInfo GetFileSystemInfo(string relativePath) |
373 |
{ |
374 |
if (String.IsNullOrWhiteSpace(relativePath)) |
375 |
throw new ArgumentNullException("relativePath"); |
376 |
//A RootPath must be set before calling this method |
377 |
if (String.IsNullOrWhiteSpace(RootPath)) |
378 |
throw new InvalidOperationException("RootPath was not set"); |
379 |
Contract.EndContractBlock(); |
380 |
|
381 |
var absolutePath = Path.Combine(RootPath, relativePath); |
382 |
|
383 |
if (Directory.Exists(absolutePath)) |
384 |
return new DirectoryInfo(absolutePath).WithProperCapitalization(); |
385 |
else |
386 |
return new FileInfo(absolutePath).WithProperCapitalization(); |
387 |
|
388 |
} |
389 |
|
390 |
public void Delete(string relativePath) |
391 |
{ |
392 |
var absolutePath = Path.Combine(RootPath, relativePath).ToLower(); |
393 |
if (File.Exists(absolutePath)) |
394 |
{ |
395 |
try |
396 |
{ |
397 |
File.Delete(absolutePath); |
398 |
} |
399 |
//The file may have been deleted by another thread. Just ignore the relevant exception |
400 |
catch (FileNotFoundException) { } |
401 |
} |
402 |
else if (Directory.Exists(absolutePath)) |
403 |
{ |
404 |
try |
405 |
{ |
406 |
Directory.Delete(absolutePath, true); |
407 |
} |
408 |
//The directory may have been deleted by another thread. Just ignore the relevant exception |
409 |
catch (DirectoryNotFoundException){} |
410 |
} |
411 |
|
412 |
//_ignoreFiles[absolutePath] = absolutePath; |
413 |
StatusKeeper.ClearFileStatus(absolutePath); |
414 |
} |
415 |
} |
416 |
} |