2 /* -----------------------------------------------------------------------
3 * <copyright file="FileAgent.cs" company="GRNet">
5 * Copyright 2011-2012 GRNET S.A. All rights reserved.
7 * Redistribution and use in source and binary forms, with or
8 * without modification, are permitted provided that the following
11 * 1. Redistributions of source code must retain the above
12 * copyright notice, this list of conditions and the following
15 * 2. Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer in the documentation and/or other materials
18 * provided with the distribution.
21 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
34 * The views and conclusions contained in the software and
35 * documentation are those of the authors and should not be
36 * interpreted as representing official policies, either expressed
37 * or implied, of GRNET S.A.
39 * -----------------------------------------------------------------------
43 using System.Collections.Generic;
44 using System.Diagnostics.Contracts;
47 using System.Reflection;
48 using System.Threading.Tasks;
49 using Pithos.Interfaces;
53 namespace Pithos.Core.Agents
56 public class FileAgent
58 private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
60 Agent<WorkflowState> _agent;
61 private FileSystemWatcher _watcher;
62 private FileSystemWatcherAdapter _adapter;
65 public IStatusKeeper StatusKeeper { get; set; }
67 public IStatusNotification StatusNotification { get; set; }
69 public IPithosWorkflow Workflow { get; set; }
71 public WorkflowAgent WorkflowAgent { get; set; }
73 private AccountInfo AccountInfo { get; set; }
75 internal string RootPath { get; set; }
77 private FileEventIdleBatch _eventIdleBatch;
79 public TimeSpan IdleTimeout { get; set; }
82 private void ProcessBatchedEvents(Dictionary<string, FileSystemEventArgs[]> fileEvents)
84 StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Uploading {0} files",fileEvents.Count));
85 foreach (var fileEvent in fileEvents)
87 var filePath = fileEvent.Key;
88 var changes = fileEvent.Value;
90 if (Ignore(filePath)) continue;
92 foreach (var change in changes)
94 if (change.ChangeType == WatcherChangeTypes.Renamed)
96 var rename = (MovedEventArgs) change;
97 _agent.Post(new WorkflowState
99 AccountInfo = AccountInfo,
100 OldPath = rename.OldFullPath,
101 OldFileName = Path.GetFileName(rename.OldName),
102 Path = rename.FullPath,
103 FileName = Path.GetFileName(rename.Name),
104 TriggeringChange = rename.ChangeType
108 _agent.Post(new WorkflowState
110 AccountInfo = AccountInfo,
111 Path = change.FullPath,
112 FileName = Path.GetFileName(change.Name),
113 TriggeringChange = change.ChangeType
117 StatusNotification.SetPithosStatus(PithosStatus.LocalComplete);
120 public void Start(AccountInfo accountInfo,string rootPath)
122 if (accountInfo==null)
123 throw new ArgumentNullException("accountInfo");
124 if (String.IsNullOrWhiteSpace(rootPath))
125 throw new ArgumentNullException("rootPath");
126 if (!Path.IsPathRooted(rootPath))
127 throw new ArgumentException("rootPath must be an absolute path","rootPath");
128 if (IdleTimeout == null)
129 throw new InvalidOperationException("IdleTimeout must have a valid value");
130 Contract.EndContractBlock();
132 AccountInfo = accountInfo;
135 _eventIdleBatch = new FileEventIdleBatch((int)IdleTimeout.TotalMilliseconds, ProcessBatchedEvents);
137 _watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true,InternalBufferSize=8*4096};
138 _adapter = new FileSystemWatcherAdapter(_watcher);
140 _adapter.Changed += OnFileEvent;
141 _adapter.Created += OnFileEvent;
142 _adapter.Deleted += OnFileEvent;
143 //_adapter.Renamed += OnRenameEvent;
144 _adapter.Moved += OnMoveEvent;
145 _watcher.EnableRaisingEvents = true;
148 _agent = Agent<WorkflowState>.Start(inbox =>
153 var message = inbox.Receive();
154 var process=message.Then(Process,inbox.CancellationToken);
155 inbox.LoopAsync(process,loop,ex=>
156 Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
162 private Task<object> Process(WorkflowState state)
165 throw new ArgumentNullException("state");
166 Contract.EndContractBlock();
168 if (Ignore(state.Path))
169 return CompletedTask<object>.Default;
171 var networkState = NetworkGate.GetNetworkState(state.Path);
172 //Skip if the file is already being downloaded or uploaded and
173 //the change is create or modify
174 if (networkState != NetworkOperation.None &&
176 state.TriggeringChange == WatcherChangeTypes.Created ||
177 state.TriggeringChange == WatcherChangeTypes.Changed
179 return CompletedTask<object>.Default;
183 //StatusKeeper.EnsureFileState(state.Path);
185 UpdateFileStatus(state);
186 UpdateOverlayStatus(state);
187 UpdateFileChecksum(state);
188 WorkflowAgent.Post(state);
190 catch (IOException exc)
192 if (File.Exists(state.Path))
194 Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
199 Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
202 catch (Exception exc)
204 Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
207 return CompletedTask<object>.Default;
212 get { return _watcher == null || !_watcher.EnableRaisingEvents; }
215 if (_watcher != null)
216 _watcher.EnableRaisingEvents = !value;
220 public string CachePath { get; set; }
222 private List<string> _selectivePaths = new List<string>();
223 public List<string> SelectivePaths
225 get { return _selectivePaths; }
226 set { _selectivePaths = value; }
230 public void Post(WorkflowState workflowState)
232 if (workflowState == null)
233 throw new ArgumentNullException("workflowState");
234 Contract.EndContractBlock();
236 _agent.Post(workflowState);
241 if (_watcher != null)
251 // Enumerate all files in the Pithos directory except those in the Fragment folder
252 // and files with a .ignore extension
253 public IEnumerable<string> EnumerateFiles(string searchPattern="*")
255 var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
256 where !Ignore(filePath)
258 return monitoredFiles;
261 public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
263 var rootDir = new DirectoryInfo(RootPath);
264 var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
265 where !Ignore(file.FullName)
267 return monitoredFiles;
270 public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
272 var rootDir = new DirectoryInfo(RootPath);
273 var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
274 where !Ignore(file.FullName)
275 select file.AsRelativeUrlTo(RootPath);
276 return monitoredFiles;
279 public IEnumerable<string> EnumerateFilesSystemInfosAsRelativeUrls(string searchPattern="*")
281 var rootDir = new DirectoryInfo(RootPath);
282 var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories)
283 where !Ignore(file.FullName)
284 select file.AsRelativeUrlTo(RootPath);
285 return monitoredFiles;
291 private bool Ignore(string filePath)
293 //Ignore all first-level directories and files (ie at the container folders level)
294 if (FoundBelowRoot(filePath, RootPath,1))
297 //Ignore first-level items under the "others" folder (ie at the accounts folders level).
298 var othersPath = Path.Combine(RootPath, FolderConstants.OthersFolder);
299 if (FoundBelowRoot(filePath, othersPath,1))
302 //Ignore second-level (container) folders under the "others" folder (ie at the container folders level).
303 if (FoundBelowRoot(filePath, othersPath,2))
307 //Ignore anything happening in the cache path
308 if (filePath.StartsWith(CachePath))
310 if (_ignoreFiles.ContainsKey(filePath.ToLower()))
313 //Ignore if selective synchronization is defined,
314 return SelectivePaths.Count > 0
315 //And the target file is not below any of the selective paths
316 && !SelectivePaths.Any(filePath.IsAtOrDirectlyBelow);
319 /* private static bool FoundInRoot(string filePath, string rootPath)
321 //var rootDirectory = new DirectoryInfo(rootPath);
323 //If the paths are equal, return true
324 if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
327 //If the filepath is below the root path
328 if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
330 //Get the relative path
331 var relativePath = filePath.Substring(rootPath.Length + 1);
332 //If the relativePath does NOT contains a path separator, we found a match
333 return (!relativePath.Contains(@"\"));
336 //If the filepath is not under the root path, return false
341 private static bool FoundBelowRoot(string filePath, string rootPath,int level)
343 //var rootDirectory = new DirectoryInfo(rootPath);
345 //If the paths are equal, return true
346 if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
349 //If the filepath is below the root path
350 if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
352 //Get the relative path
353 var relativePath = filePath.Substring(rootPath.Length + 1);
354 //If the relativePath does NOT contains a path separator, we found a match
355 var levels=relativePath.ToCharArray().Count(c=>c=='\\')+1;
356 return levels==level;
359 //If the filepath is not under the root path, return false
363 //Post a Change message for all events except rename
364 void OnFileEvent(object sender, FileSystemEventArgs e)
366 //Ignore events that affect the cache folder
367 var filePath = e.FullPath;
368 if (Ignore(filePath))
370 _eventIdleBatch.Post(e);
375 //Post a Change message for renames containing the old and new names
376 void OnRenameEvent(object sender, RenamedEventArgs e)
378 var oldFullPath = e.OldFullPath;
379 var fullPath = e.FullPath;
380 if (Ignore(oldFullPath) || Ignore(fullPath))
383 _agent.Post(new WorkflowState
385 AccountInfo=AccountInfo,
386 OldPath = oldFullPath,
387 OldFileName = e.OldName,
390 TriggeringChange = e.ChangeType
395 //Post a Change message for moves containing the old and new names
396 void OnMoveEvent(object sender, MovedEventArgs e)
398 var oldFullPath = e.OldFullPath;
399 var fullPath = e.FullPath;
400 if (Ignore(oldFullPath) || Ignore(fullPath))
403 _eventIdleBatch.Post(e);
408 private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
410 {WatcherChangeTypes.Created,FileStatus.Created},
411 {WatcherChangeTypes.Changed,FileStatus.Modified},
412 {WatcherChangeTypes.Deleted,FileStatus.Deleted},
413 {WatcherChangeTypes.Renamed,FileStatus.Renamed}
416 private Dictionary<string, string> _ignoreFiles=new Dictionary<string, string>();
418 private WorkflowState UpdateFileStatus(WorkflowState state)
421 throw new ArgumentNullException("state");
422 if (String.IsNullOrWhiteSpace(state.Path))
423 throw new ArgumentException("The state's Path can't be empty","state");
424 Contract.EndContractBlock();
426 var path = state.Path;
427 var status = _statusDict[state.TriggeringChange];
428 var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
429 if (status == oldStatus)
431 state.Status = status;
435 if (state.Status == FileStatus.Renamed)
436 Workflow.ClearFileStatus(path);
438 state.Status = Workflow.SetFileStatus(path, status);
442 private WorkflowState UpdateOverlayStatus(WorkflowState state)
445 throw new ArgumentNullException("state");
446 Contract.EndContractBlock();
451 switch (state.Status)
453 case FileStatus.Created:
454 this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash);
456 case FileStatus.Modified:
457 this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash);
459 case FileStatus.Deleted:
460 //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
462 case FileStatus.Renamed:
463 this.StatusKeeper.ClearFileStatus(state.OldPath);
464 this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash);
466 case FileStatus.Unchanged:
467 this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal,state.ShortHash);
471 if (state.Status == FileStatus.Deleted)
472 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
474 NativeMethods.RaiseChangeNotification(state.Path);
479 private WorkflowState UpdateFileChecksum(WorkflowState state)
484 if (state.Status == FileStatus.Deleted)
487 var path = state.Path;
488 //Skip calculation for folders
489 if (Directory.Exists(path))
493 var info = new FileInfo(path);
494 StatusNotification.Notify(new StatusNotification(String.Format("Hashing [{0}]",info.Name)));
496 var shortHash = info.ComputeShortHash();
498 string merkleHash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
499 StatusKeeper.UpdateFileChecksum(path,shortHash, merkleHash);
501 state.Hash = merkleHash;
505 //Does the file exist in the container's local folder?
506 public bool Exists(string relativePath)
508 if (String.IsNullOrWhiteSpace(relativePath))
509 throw new ArgumentNullException("relativePath");
510 //A RootPath must be set before calling this method
511 if (String.IsNullOrWhiteSpace(RootPath))
512 throw new InvalidOperationException("RootPath was not set");
513 Contract.EndContractBlock();
514 //Create the absolute path by combining the RootPath with the relativePath
515 var absolutePath=Path.Combine(RootPath, relativePath);
516 //Is this a valid file?
517 if (File.Exists(absolutePath))
520 if (Directory.Exists(absolutePath))
522 //Fail if it is neither
526 public static FileAgent GetFileAgent(AccountInfo accountInfo)
528 return GetFileAgent(accountInfo.AccountPath);
531 public static FileAgent GetFileAgent(string rootPath)
533 return AgentLocator<FileAgent>.Get(rootPath.ToLower());
537 public FileSystemInfo GetFileSystemInfo(string relativePath)
539 if (String.IsNullOrWhiteSpace(relativePath))
540 throw new ArgumentNullException("relativePath");
541 //A RootPath must be set before calling this method
542 if (String.IsNullOrWhiteSpace(RootPath))
543 throw new InvalidOperationException("RootPath was not set");
544 Contract.EndContractBlock();
546 var absolutePath = Path.Combine(RootPath, relativePath);
548 if (Directory.Exists(absolutePath))
549 return new DirectoryInfo(absolutePath).WithProperCapitalization();
551 return new FileInfo(absolutePath).WithProperCapitalization();
555 public void Delete(string relativePath)
557 var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
558 if (Log.IsDebugEnabled)
559 Log.DebugFormat("Deleting {0}", absolutePath);
560 if (File.Exists(absolutePath))
564 File.Delete(absolutePath);
566 //The file may have been deleted by another thread. Just ignore the relevant exception
567 catch (FileNotFoundException) { }
569 else if (Directory.Exists(absolutePath))
573 Directory.Delete(absolutePath, true);
575 //The directory may have been deleted by another thread. Just ignore the relevant exception
576 catch (DirectoryNotFoundException){}
579 //_ignoreFiles[absolutePath] = absolutePath;
580 StatusKeeper.ClearFileStatus(absolutePath);