2 /* -----------------------------------------------------------------------
3 * <copyright file="FileAgent.cs" company="GRNet">
5 * Copyright 2011-2012 GRNET S.A. All rights reserved.
7 * Redistribution and use in source and binary forms, with or
8 * without modification, are permitted provided that the following
11 * 1. Redistributions of source code must retain the above
12 * copyright notice, this list of conditions and the following
15 * 2. Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer in the documentation and/or other materials
18 * provided with the distribution.
21 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
34 * The views and conclusions contained in the software and
35 * documentation are those of the authors and should not be
36 * interpreted as representing official policies, either expressed
37 * or implied, of GRNET S.A.
39 * -----------------------------------------------------------------------
43 using System.Collections.Generic;
44 using System.Diagnostics.Contracts;
47 using System.Reflection;
48 using System.Threading.Tasks;
49 using Pithos.Interfaces;
53 namespace Pithos.Core.Agents
56 public class FileAgent
58 private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
61 Agent<WorkflowState> _agent;
63 private FileSystemWatcher _watcher;
64 private FileSystemWatcherAdapter _adapter;
65 private FileEventIdleBatch _eventIdleBatch;
68 public IStatusKeeper StatusKeeper { get; set; }
70 public IStatusNotification StatusNotification { get; set; }
72 public IPithosWorkflow Workflow { get; set; }
74 //public WorkflowAgent WorkflowAgent { get; set; }
76 private AccountInfo AccountInfo { get; set; }
78 internal string RootPath { get; set; }
80 public TimeSpan IdleTimeout { get; set; }
82 public PollAgent PollAgent { get; set; }
84 private void ProcessBatchedEvents(Dictionary<string, FileSystemEventArgs[]> fileEvents)
86 var paths = fileEvents.Keys;
88 PollAgent.SynchNow(/*paths*/);
92 private void ProcessBatchedEvents(Dictionary<string, FileSystemEventArgs[]> fileEvents)
94 StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Uploading {0} files",fileEvents.Count));
95 //Start with events that do not originate in one of the ignored folders
96 var initialEvents = from evt in fileEvents
97 where !IgnorePaths(evt.Key)
100 IEnumerable<KeyValuePair<string, FileSystemEventArgs[]>> cleanEvents;
103 var selectiveEnabled = Selectives.IsSelectiveEnabled(AccountInfo.AccountKey);
104 //When selective sync is enabled,
105 if (selectiveEnabled)
107 //Include all selected items
108 var selectedEvents = from evt in initialEvents
109 where Selectives.IsSelected(AccountInfo, evt.Key)
111 //And all folder creations in the unselected folders
112 var folderCreations = from evt in initialEvents
113 let folderPath=evt.Key
114 //The original folder may not exist due to renames. Just make sure that the path is not a file
115 where !File.Exists(folderPath)
116 //We only want unselected items
117 && !Selectives.IsSelected(AccountInfo, folderPath)
118 //Is there any creation event related to the folder?
119 && evt.Value.Any(arg => arg.ChangeType == WatcherChangeTypes.Created)
121 cleanEvents = selectedEvents.Union(folderCreations).ToList();
123 //If selective is disabled, only exclude the shared folders
126 cleanEvents = (from evt in initialEvents
127 where !evt.Key.IsSharedTo(AccountInfo)
128 select evt).ToList();
132 foreach (var fileEvent in cleanEvents)
134 //var filePath = fileEvent.Key;
135 var changes = fileEvent.Value;
137 var isNotFile = !File.Exists(fileEvent.Key);
138 foreach (var change in changes)
140 if (change.ChangeType == WatcherChangeTypes.Renamed)
142 var rename = (MovedEventArgs) change;
143 _agent.Post(new WorkflowState(change)
145 AccountInfo = AccountInfo,
146 OldPath = rename.OldFullPath,
147 OldFileName = Path.GetFileName(rename.OldName),
148 Path = rename.FullPath,
149 FileName = Path.GetFileName(rename.Name),
150 TriggeringChange = rename.ChangeType
155 var isCreation = selectiveEnabled && isNotFile && change.ChangeType == WatcherChangeTypes.Created;
156 _agent.Post(new WorkflowState(change)
158 AccountInfo = AccountInfo,
159 Path = change.FullPath,
160 FileName = Path.GetFileName(change.Name),
161 TriggeringChange = change.ChangeType,
162 IsCreation=isCreation
167 StatusNotification.SetPithosStatus(PithosStatus.LocalComplete);
171 public void Start(AccountInfo accountInfo,string rootPath)
173 if (accountInfo==null)
174 throw new ArgumentNullException("accountInfo");
175 if (String.IsNullOrWhiteSpace(rootPath))
176 throw new ArgumentNullException("rootPath");
177 if (!Path.IsPathRooted(rootPath))
178 throw new ArgumentException("rootPath must be an absolute path","rootPath");
179 if (IdleTimeout == null)
180 throw new InvalidOperationException("IdleTimeout must have a valid value");
181 Contract.EndContractBlock();
183 AccountInfo = accountInfo;
186 _eventIdleBatch = new FileEventIdleBatch((int)IdleTimeout.TotalMilliseconds, ProcessBatchedEvents);
188 _watcher = new FileSystemWatcher(rootPath) { IncludeSubdirectories = true, InternalBufferSize = 8 * 4096 };
189 _adapter = new FileSystemWatcherAdapter(_watcher);
191 _adapter.Changed += OnFileEvent;
192 _adapter.Created += OnFileEvent;
193 _adapter.Deleted += OnFileEvent;
194 //_adapter.Renamed += OnRenameEvent;
195 _adapter.Moved += OnMoveEvent;
196 _watcher.EnableRaisingEvents = true;
202 _agent = Agent<WorkflowState>.Start(inbox =>
207 var message = inbox.Receive();
208 var process=message.Then(Process,inbox.CancellationToken);
209 inbox.LoopAsync(process,loop,ex=>
210 Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
217 private Task<object> Process(WorkflowState state)
220 throw new ArgumentNullException("state");
221 Contract.EndContractBlock();
223 if (Ignore(state.Path))
224 return CompletedTask<object>.Default;
226 var networkState = NetworkGate.GetNetworkState(state.Path);
227 //Skip if the file is already being downloaded or uploaded and
228 //the change is create or modify
229 if (networkState != NetworkOperation.None &&
231 state.TriggeringChange == WatcherChangeTypes.Created ||
232 state.TriggeringChange == WatcherChangeTypes.Changed
234 return CompletedTask<object>.Default;
238 //StatusKeeper.EnsureFileState(state.Path);
240 UpdateFileStatus(state);
241 UpdateOverlayStatus(state);
242 UpdateFileChecksum(state);
243 WorkflowAgent.Post(state);
245 catch (IOException exc)
247 if (File.Exists(state.Path))
249 Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
254 Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
257 catch (Exception exc)
259 Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
262 return CompletedTask<object>.Default;
267 get { return _watcher == null || !_watcher.EnableRaisingEvents; }
270 if (_watcher != null)
271 _watcher.EnableRaisingEvents = !value;
276 public string CachePath { get; set; }
278 /*private List<string> _selectivePaths = new List<string>();
279 public List<string> SelectivePaths
281 get { return _selectivePaths; }
282 set { _selectivePaths = value; }
285 public Selectives Selectives { get; set; }
289 public void Post(WorkflowState workflowState)
291 if (workflowState == null)
292 throw new ArgumentNullException("workflowState");
293 Contract.EndContractBlock();
295 _agent.Post(workflowState);
300 if (_watcher != null)
311 // Enumerate all files in the Pithos directory except those in the Fragment folder
312 // and files with a .ignore extension
313 public IEnumerable<string> EnumerateFiles(string searchPattern="*")
315 var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
316 where !Ignore(filePath)
317 orderby filePath ascending
319 return monitoredFiles;
322 public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
324 var rootDir = new DirectoryInfo(RootPath);
325 var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
326 where !Ignore(file.FullName)
327 orderby file.FullName ascending
329 return monitoredFiles;
332 public IEnumerable<FileSystemInfo> EnumerateFileSystemInfos(string searchPattern="*")
334 var rootDir = new DirectoryInfo(RootPath);
335 //Ensure folders appear first, to allow folder processing as soon as possilbe
336 var folders = (from file in rootDir.EnumerateDirectories(searchPattern, SearchOption.AllDirectories)
337 where !Ignore(file.FullName)
338 orderby file.FullName ascending
339 select file).ToList();
340 var files = (from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
341 where !Ignore(file.FullName)
342 orderby file.Length ascending
343 select file as FileSystemInfo).ToList();
344 var monitoredFiles = folders
345 //Process small files first, leaving expensive large files for last
347 return monitoredFiles;
350 public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
352 var rootDir = new DirectoryInfo(RootPath);
353 var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
354 where !Ignore(file.FullName)
355 orderby file.FullName ascending
356 select file.AsRelativeUrlTo(RootPath);
357 return monitoredFiles;
360 public IEnumerable<string> EnumerateFilesSystemInfosAsRelativeUrls(string searchPattern="*")
362 var rootDir = new DirectoryInfo(RootPath);
363 var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories)
364 where !Ignore(file.FullName)
365 orderby file.FullName ascending
366 select file.AsRelativeUrlTo(RootPath);
367 return monitoredFiles;
373 public bool Ignore(string filePath)
375 if (IgnorePaths(filePath)) return true;
378 //If selective sync is enabled,
379 if (IsUnselectedRootFolder(filePath))
381 //Ignore if selective synchronization is defined,
382 //And the target file is not below any of the selective paths
383 var ignore = !Selectives.IsSelected(AccountInfo, filePath);
387 public bool IsUnselectedRootFolder(string filePath)
389 return Selectives.IsSelectiveEnabled(AccountInfo.AccountKey) //propagate folder events
390 && Directory.Exists(filePath) //from the container root folder only. Note, in the first level below the account root path are the containers
391 && FoundBelowRoot(filePath, RootPath, 2);
394 public bool IgnorePaths(string filePath)
396 //Ignore all first-level directories and files (ie at the container folders level)
397 if (FoundBelowRoot(filePath, RootPath, 1))
400 //Ignore first-level items under the "others" folder (ie at the accounts folders level).
401 var othersPath = Path.Combine(RootPath, FolderConstants.OthersFolder);
402 if (FoundBelowRoot(filePath, othersPath, 1))
405 //Ignore second-level (container) folders under the "others" folder (ie at the container folders level).
406 if (FoundBelowRoot(filePath, othersPath, 2))
410 //Ignore anything happening in the cache path
411 if (filePath.StartsWith(CachePath))
414 //Finally, ignore events about one of the ignored files
415 return _ignoreFiles.ContainsKey(filePath.ToLower());
418 /* private static bool FoundInRoot(string filePath, string rootPath)
420 //var rootDirectory = new DirectoryInfo(rootPath);
422 //If the paths are equal, return true
423 if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
426 //If the filepath is below the root path
427 if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
429 //Get the relative path
430 var relativePath = filePath.Substring(rootPath.Length + 1);
431 //If the relativePath does NOT contains a path separator, we found a match
432 return (!relativePath.Contains(@"\"));
435 //If the filepath is not under the root path, return false
440 private static bool FoundBelowRoot(string filePath, string rootPath,int level)
442 //var rootDirectory = new DirectoryInfo(rootPath);
444 //If the paths are equal, return true
445 if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
448 //If the filepath is below the root path
449 if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
451 //Get the relative path
452 var relativePath = filePath.Substring(rootPath.Length + 1);
453 //If the relativePath does NOT contains a path separator, we found a match
454 var levels=relativePath.ToCharArray().Count(c=>c=='\\')+1;
455 return levels==level;
458 //If the filepath is not under the root path, return false
463 //Post a Change message for renames containing the old and new names
464 void OnRenameEvent(object sender, RenamedEventArgs e)
466 var oldFullPath = e.OldFullPath;
467 var fullPath = e.FullPath;
468 if (Ignore(oldFullPath) || Ignore(fullPath))
471 _agent.Post(new WorkflowState
473 AccountInfo=AccountInfo,
474 OldPath = oldFullPath,
475 OldFileName = e.OldName,
478 TriggeringChange = e.ChangeType
483 //Post a Change message for all events except rename
484 void OnFileEvent(object sender, FileSystemEventArgs e)
486 //Ignore events that affect the cache folder
487 var filePath = e.FullPath;
488 if (Ignore(filePath))
490 _eventIdleBatch.Post(e);
493 //Post a Change message for moves containing the old and new names
494 void OnMoveEvent(object sender, MovedEventArgs e)
496 var oldFullPath = e.OldFullPath;
497 var fullPath = e.FullPath;
500 //If the source path is one of the ignored folders, ignore
501 if (IgnorePaths(oldFullPath))
504 //TODO: Must prevent move propagation if the source folder is blocked by selective sync
505 //Ignore takes into account Selective Sync
506 if (Ignore(fullPath))
509 _eventIdleBatch.Post(e);
514 private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
516 {WatcherChangeTypes.Created,FileStatus.Created},
517 {WatcherChangeTypes.Changed,FileStatus.Modified},
518 {WatcherChangeTypes.Deleted,FileStatus.Deleted},
519 {WatcherChangeTypes.Renamed,FileStatus.Renamed}
522 private Dictionary<string, string> _ignoreFiles=new Dictionary<string, string>();
524 private WorkflowState UpdateFileStatus(WorkflowState state)
527 throw new ArgumentNullException("state");
528 if (String.IsNullOrWhiteSpace(state.Path))
529 throw new ArgumentException("The state's Path can't be empty","state");
530 Contract.EndContractBlock();
532 var path = state.Path;
533 var status = _statusDict[state.TriggeringChange];
534 var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
535 if (status == oldStatus)
537 state.Status = status;
541 if (state.Status == FileStatus.Renamed)
542 Workflow.ClearFileStatus(path);
544 state.Status = Workflow.SetFileStatus(path, status);
548 private WorkflowState UpdateOverlayStatus(WorkflowState state)
551 throw new ArgumentNullException("state");
552 Contract.EndContractBlock();
557 switch (state.Status)
559 case FileStatus.Created:
560 this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash).Wait();
562 case FileStatus.Modified:
563 this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified, state.ShortHash).Wait();
565 case FileStatus.Deleted:
566 //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
568 case FileStatus.Renamed:
569 this.StatusKeeper.ClearFileStatus(state.OldPath);
570 this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified, state.ShortHash).Wait();
572 case FileStatus.Unchanged:
573 this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal, state.ShortHash).Wait();
577 if (state.Status == FileStatus.Deleted)
578 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
580 NativeMethods.RaiseChangeNotification(state.Path);
585 private WorkflowState UpdateFileChecksum(WorkflowState state)
590 if (state.Status == FileStatus.Deleted)
593 var path = state.Path;
594 //Skip calculation for folders
595 if (Directory.Exists(path))
598 var info = new FileInfo(path);
600 using (StatusNotification.GetNotifier("Hashing {0}", "Finished Hashing {0}", info.Name))
603 var shortHash = info.ComputeShortHash();
605 string merkleHash = info.CalculateHash(StatusKeeper.BlockSize, StatusKeeper.BlockHash);
606 StatusKeeper.UpdateFileChecksum(path, shortHash, merkleHash);
608 state.Hash = merkleHash;
613 //Does the file exist in the container's local folder?
614 public bool Exists(string relativePath)
616 if (String.IsNullOrWhiteSpace(relativePath))
617 throw new ArgumentNullException("relativePath");
618 //A RootPath must be set before calling this method
619 if (String.IsNullOrWhiteSpace(RootPath))
620 throw new InvalidOperationException("RootPath was not set");
621 Contract.EndContractBlock();
622 //Create the absolute path by combining the RootPath with the relativePath
623 var absolutePath=Path.Combine(RootPath, relativePath);
624 //Is this a valid file?
625 if (File.Exists(absolutePath))
628 if (Directory.Exists(absolutePath))
630 //Fail if it is neither
634 public static FileAgent GetFileAgent(AccountInfo accountInfo)
636 return GetFileAgent(accountInfo.AccountPath);
639 public static FileAgent GetFileAgent(string rootPath)
641 return AgentLocator<FileAgent>.Get(rootPath.ToLower());
645 public FileSystemInfo GetFileSystemInfo(string relativePath)
647 if (String.IsNullOrWhiteSpace(relativePath))
648 throw new ArgumentNullException("relativePath");
649 //A RootPath must be set before calling this method
650 if (String.IsNullOrWhiteSpace(RootPath))
651 throw new InvalidOperationException("RootPath was not set");
652 Contract.EndContractBlock();
654 var absolutePath = Path.Combine(RootPath, relativePath);
656 if (Directory.Exists(absolutePath))
657 return new DirectoryInfo(absolutePath).WithProperCapitalization();
659 return new FileInfo(absolutePath).WithProperCapitalization();
663 public void Delete(string relativePath)
665 var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
666 if (Log.IsDebugEnabled)
667 Log.DebugFormat("Deleting {0}", absolutePath);
668 if (File.Exists(absolutePath))
672 File.Delete(absolutePath);
674 //The file may have been deleted by another thread. Just ignore the relevant exception
675 catch (FileNotFoundException) { }
677 else if (Directory.Exists(absolutePath))
681 Directory.Delete(absolutePath, true);
683 //The directory may have been deleted by another thread. Just ignore the relevant exception
684 catch (DirectoryNotFoundException){}
687 //_ignoreFiles[absolutePath] = absolutePath;
688 StatusKeeper.ClearFileStatus(absolutePath);