2 /* -----------------------------------------------------------------------
3 * <copyright file="FileAgent.cs" company="GRNet">
5 * Copyright 2011-2012 GRNET S.A. All rights reserved.
7 * Redistribution and use in source and binary forms, with or
8 * without modification, are permitted provided that the following
11 * 1. Redistributions of source code must retain the above
12 * copyright notice, this list of conditions and the following
15 * 2. Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer in the documentation and/or other materials
18 * provided with the distribution.
21 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
34 * The views and conclusions contained in the software and
35 * documentation are those of the authors and should not be
36 * interpreted as representing official policies, either expressed
37 * or implied, of GRNET S.A.
39 * -----------------------------------------------------------------------
43 using System.Collections.Generic;
44 using System.Diagnostics.Contracts;
47 using System.Reflection;
48 using System.Threading.Tasks;
49 using Pithos.Interfaces;
53 namespace Pithos.Core.Agents
56 public class FileAgent
58 private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
61 Agent<WorkflowState> _agent;
63 private FileSystemWatcher _watcher;
64 private FileSystemWatcherAdapter _adapter;
65 private FileEventIdleBatch _eventIdleBatch;
68 public IStatusKeeper StatusKeeper { get; set; }
70 public IStatusNotification StatusNotification { get; set; }
72 public IPithosWorkflow Workflow { get; set; }
74 //public WorkflowAgent WorkflowAgent { get; set; }
76 private AccountInfo AccountInfo { get; set; }
78 internal string RootPath { get; set; }
80 public TimeSpan IdleTimeout { get; set; }
82 public PollAgent PollAgent { get; set; }
84 private void ProcessBatchedEvents(FileSystemEventArgs[] fileEvents)
86 var paths = new HashSet<string>();
88 foreach (var evt in fileEvents)
90 paths.Add(evt.FullPath);
91 if (evt is MovedEventArgs)
93 paths.Add((evt as MovedEventArgs).OldFullPath);
95 else if (evt is RenamedEventArgs)
97 paths.Add((evt as RenamedEventArgs).OldFullPath);
101 PollAgent.SynchNow(paths);
105 private void ProcessBatchedEvents(Dictionary<string, FileSystemEventArgs[]> fileEvents)
107 StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Uploading {0} files",fileEvents.Count));
108 //Start with events that do not originate in one of the ignored folders
109 var initialEvents = from evt in fileEvents
110 where !IgnorePaths(evt.Key)
113 IEnumerable<KeyValuePair<string, FileSystemEventArgs[]>> cleanEvents;
116 var selectiveEnabled = Selectives.IsSelectiveEnabled(AccountInfo.AccountKey);
117 //When selective sync is enabled,
118 if (selectiveEnabled)
120 //Include all selected items
121 var selectedEvents = from evt in initialEvents
122 where Selectives.IsSelected(AccountInfo, evt.Key)
124 //And all folder creations in the unselected folders
125 var folderCreations = from evt in initialEvents
126 let folderPath=evt.Key
127 //The original folder may not exist due to renames. Just make sure that the path is not a file
128 where !File.Exists(folderPath)
129 //We only want unselected items
130 && !Selectives.IsSelected(AccountInfo, folderPath)
131 //Is there any creation event related to the folder?
132 && evt.Value.Any(arg => arg.ChangeType == WatcherChangeTypes.Created)
134 cleanEvents = selectedEvents.Union(folderCreations).ToList();
136 //If selective is disabled, only exclude the shared folders
139 cleanEvents = (from evt in initialEvents
140 where !evt.Key.IsSharedTo(AccountInfo)
141 select evt).ToList();
145 foreach (var fileEvent in cleanEvents)
147 //var filePath = fileEvent.Key;
148 var changes = fileEvent.Value;
150 var isNotFile = !File.Exists(fileEvent.Key);
151 foreach (var change in changes)
153 if (change.ChangeType == WatcherChangeTypes.Renamed)
155 var rename = (MovedEventArgs) change;
156 _agent.Post(new WorkflowState(change)
158 AccountInfo = AccountInfo,
159 OldPath = rename.OldFullPath,
160 OldFileName = Path.GetFileName(rename.OldName),
161 Path = rename.FullPath,
162 FileName = Path.GetFileName(rename.Name),
163 TriggeringChange = rename.ChangeType
168 var isCreation = selectiveEnabled && isNotFile && change.ChangeType == WatcherChangeTypes.Created;
169 _agent.Post(new WorkflowState(change)
171 AccountInfo = AccountInfo,
172 Path = change.FullPath,
173 FileName = Path.GetFileName(change.Name),
174 TriggeringChange = change.ChangeType,
175 IsCreation=isCreation
180 StatusNotification.SetPithosStatus(PithosStatus.LocalComplete);
184 public void Start(AccountInfo accountInfo,string rootPath)
186 if (accountInfo==null)
187 throw new ArgumentNullException("accountInfo");
188 if (String.IsNullOrWhiteSpace(rootPath))
189 throw new ArgumentNullException("rootPath");
190 if (!Path.IsPathRooted(rootPath))
191 throw new ArgumentException("rootPath must be an absolute path","rootPath");
192 if (IdleTimeout == null)
193 throw new InvalidOperationException("IdleTimeout must have a valid value");
194 Contract.EndContractBlock();
196 AccountInfo = accountInfo;
199 _eventIdleBatch = new FileEventIdleBatch(PollAgent,(int)IdleTimeout.TotalMilliseconds, ProcessBatchedEvents);
201 _watcher = new FileSystemWatcher(rootPath) { IncludeSubdirectories = true, InternalBufferSize = 8 * 4096 };
202 _adapter = new FileSystemWatcherAdapter(_watcher,this);
204 _adapter.Changed += OnFileEvent;
205 _adapter.Created += OnFileEvent;
206 _adapter.Deleted += OnFileEvent;
207 //_adapter.Renamed += OnRenameEvent;
208 _adapter.Moved += OnMoveEvent;
209 _watcher.EnableRaisingEvents = true;
215 _agent = Agent<WorkflowState>.Start(inbox =>
220 var message = inbox.Receive();
221 var process=message.Then(Process,inbox.CancellationToken);
222 inbox.LoopAsync(process,loop,ex=>
223 Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
230 private Task<object> Process(WorkflowState state)
233 throw new ArgumentNullException("state");
234 Contract.EndContractBlock();
236 if (Ignore(state.Path))
237 return CompletedTask<object>.Default;
239 var networkState = NetworkGate.GetNetworkState(state.Path);
240 //Skip if the file is already being downloaded or uploaded and
241 //the change is create or modify
242 if (networkState != NetworkOperation.None &&
244 state.TriggeringChange == WatcherChangeTypes.Created ||
245 state.TriggeringChange == WatcherChangeTypes.Changed
247 return CompletedTask<object>.Default;
251 //StatusKeeper.EnsureFileState(state.Path);
253 UpdateFileStatus(state);
254 UpdateOverlayStatus(state);
255 UpdateLastMD5(state);
256 WorkflowAgent.Post(state);
258 catch (IOException exc)
260 if (File.Exists(state.Path))
262 Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
267 Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
270 catch (Exception exc)
272 Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
275 return CompletedTask<object>.Default;
280 get { return _watcher == null || !_watcher.EnableRaisingEvents; }
283 if (_watcher != null)
284 _watcher.EnableRaisingEvents = !value;
289 public string CachePath { get; set; }
291 /*private List<string> _selectivePaths = new List<string>();
292 public List<string> SelectivePaths
294 get { return _selectivePaths; }
295 set { _selectivePaths = value; }
298 public Selectives Selectives { get; set; }
302 public void Post(WorkflowState workflowState)
304 if (workflowState == null)
305 throw new ArgumentNullException("workflowState");
306 Contract.EndContractBlock();
308 _agent.Post(workflowState);
313 if (_watcher != null)
324 // Enumerate all files in the Pithos directory except those in the Fragment folder
325 // and files with a .ignore extension
326 public IEnumerable<string> EnumerateFiles(string searchPattern="*")
328 var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
329 where !Ignore(filePath)
330 orderby filePath ascending
332 return monitoredFiles;
335 public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
337 var rootDir = new DirectoryInfo(RootPath);
338 var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
339 where !Ignore(file.FullName)
340 orderby file.FullName ascending
342 return monitoredFiles;
345 public IEnumerable<FileSystemInfo> EnumerateFileSystemInfos(string searchPattern="*")
347 var rootDir = new DirectoryInfo(RootPath);
348 //Ensure folders appear first, to allow folder processing as soon as possilbe
349 var folders = (from file in rootDir.EnumerateDirectories(searchPattern, SearchOption.AllDirectories)
350 where !Ignore(file.FullName)
351 orderby file.FullName ascending
352 select file).ToList();
353 var files = (from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
354 where !Ignore(file.FullName)
355 orderby file.Length ascending
356 select file as FileSystemInfo).ToList();
357 var monitoredFiles = folders
358 //Process small files first, leaving expensive large files for last
360 return monitoredFiles;
363 public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
365 var rootDir = new DirectoryInfo(RootPath);
366 var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
367 where !Ignore(file.FullName)
368 orderby file.FullName ascending
369 select file.AsRelativeUrlTo(RootPath);
370 return monitoredFiles;
373 public IEnumerable<string> EnumerateFilesSystemInfosAsRelativeUrls(string searchPattern="*")
375 var rootDir = new DirectoryInfo(RootPath);
376 var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories)
377 where !Ignore(file.FullName)
378 orderby file.FullName ascending
379 select file.AsRelativeUrlTo(RootPath);
380 return monitoredFiles;
386 public bool Ignore(string filePath)
388 if (IgnorePaths(filePath)) return true;
391 //If selective sync is enabled,
392 if (IsUnselectedRootFolder(filePath))
394 //Ignore if selective synchronization is defined,
395 //And the target file is not below any of the selective paths
396 var ignore = !Selectives.IsSelected(AccountInfo, filePath);
400 public bool IsUnselectedRootFolder(string filePath)
402 return Selectives.IsSelectiveEnabled(AccountInfo.AccountKey) //propagate folder events
403 && Directory.Exists(filePath) //from the container root folder only. Note, in the first level below the account root path are the containers
404 && FoundBelowRoot(filePath, RootPath, 2);
407 public bool IgnorePaths(string filePath)
409 //Ignore all first-level directories and files (ie at the container folders level)
410 if (FoundBelowRoot(filePath, RootPath, 1))
413 //Ignore first-level items under the "others" folder (ie at the accounts folders level).
414 var othersPath = Path.Combine(RootPath, FolderConstants.OthersFolder);
415 if (FoundBelowRoot(filePath, othersPath, 1))
418 //Ignore second-level (container) folders under the "others" folder (ie at the container folders level).
419 if (FoundBelowRoot(filePath, othersPath, 2))
423 //Ignore anything happening in the cache path
424 if (filePath.StartsWith(CachePath))
427 //Finally, ignore events about one of the ignored files
428 return _ignoreFiles.ContainsKey(filePath.ToLower());
431 /* private static bool FoundInRoot(string filePath, string rootPath)
433 //var rootDirectory = new DirectoryInfo(rootPath);
435 //If the paths are equal, return true
436 if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
439 //If the filepath is below the root path
440 if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
442 //Get the relative path
443 var relativePath = filePath.Substring(rootPath.Length + 1);
444 //If the relativePath does NOT contains a path separator, we found a match
445 return (!relativePath.Contains(@"\"));
448 //If the filepath is not under the root path, return false
453 private static bool FoundBelowRoot(string filePath, string rootPath,int level)
455 //var rootDirectory = new DirectoryInfo(rootPath);
457 //If the paths are equal, return true
458 if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
461 //If the filepath is below the root path
462 if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
464 //Get the relative path
465 var relativePath = filePath.Substring(rootPath.Length + 1);
466 //If the relativePath does NOT contains a path separator, we found a match
467 var levels=relativePath.ToCharArray().Count(c=>c=='\\')+1;
468 return levels==level;
471 //If the filepath is not under the root path, return false
476 //Post a Change message for renames containing the old and new names
477 void OnRenameEvent(object sender, RenamedEventArgs e)
479 var oldFullPath = e.OldFullPath;
480 var fullPath = e.FullPath;
481 if (Ignore(oldFullPath) || Ignore(fullPath))
484 _agent.Post(new WorkflowState
486 AccountInfo=AccountInfo,
487 OldPath = oldFullPath,
488 OldFileName = e.OldName,
491 TriggeringChange = e.ChangeType
496 //Post a Change message for all events except rename
497 void OnFileEvent(object sender, FileSystemEventArgs e)
499 //Ignore events that affect the cache folder
500 var filePath = e.FullPath;
501 if (Ignore(filePath))
503 _eventIdleBatch.Post(e);
506 //Post a Change message for moves containing the old and new names
507 void OnMoveEvent(object sender, MovedEventArgs e)
509 var oldFullPath = e.OldFullPath;
510 var fullPath = e.FullPath;
513 //If the source path is one of the ignored folders, ignore
514 if (IgnorePaths(oldFullPath))
517 //TODO: Must prevent move propagation if the source folder is blocked by selective sync
518 //Ignore takes into account Selective Sync
519 if (Ignore(fullPath))
521 PollAgent.PostMove(e);
522 _eventIdleBatch.Post(e);
527 private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
529 {WatcherChangeTypes.Created,FileStatus.Created},
530 {WatcherChangeTypes.Changed,FileStatus.Modified},
531 {WatcherChangeTypes.Deleted,FileStatus.Deleted},
532 {WatcherChangeTypes.Renamed,FileStatus.Renamed}
535 private Dictionary<string, string> _ignoreFiles=new Dictionary<string, string>();
537 private WorkflowState UpdateFileStatus(WorkflowState state)
540 throw new ArgumentNullException("state");
541 if (String.IsNullOrWhiteSpace(state.Path))
542 throw new ArgumentException("The state's Path can't be empty","state");
543 Contract.EndContractBlock();
545 var path = state.Path;
546 var status = _statusDict[state.TriggeringChange];
547 var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
548 if (status == oldStatus)
550 state.Status = status;
554 if (state.Status == FileStatus.Renamed)
555 Workflow.ClearFileStatus(path);
557 state.Status = Workflow.SetFileStatus(path, status);
561 private WorkflowState UpdateOverlayStatus(WorkflowState state)
564 throw new ArgumentNullException("state");
565 Contract.EndContractBlock();
570 switch (state.Status)
572 case FileStatus.Created:
573 this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ETag).Wait();
575 case FileStatus.Modified:
576 this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified, state.ETag).Wait();
578 case FileStatus.Deleted:
579 //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
581 case FileStatus.Renamed:
582 this.StatusKeeper.ClearFileStatus(state.OldPath);
583 this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified, state.ETag).Wait();
585 case FileStatus.Unchanged:
586 this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal, state.ETag).Wait();
590 if (state.Status == FileStatus.Deleted)
591 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
593 NativeMethods.RaiseChangeNotification(state.Path);
598 private WorkflowState UpdateFileChecksum(WorkflowState state)
603 if (state.Status == FileStatus.Deleted)
606 var path = state.Path;
607 //Skip calculation for folders
608 if (Directory.Exists(path))
611 var info = new FileInfo(path);
613 using (StatusNotification.GetNotifier("Hashing {0}", "Finished Hashing {0}", info.Name))
616 var etag = info.ComputeShortHash(StatusNotification);
618 var progress = new Progress<double>(d =>
619 StatusNotification.Notify(new StatusNotification(String.Format("Hashing {0} of {1}", d, info.Name))));
621 string merkleHash = info.CalculateHash(StatusKeeper.BlockSize, StatusKeeper.BlockHash,progress);
622 StatusKeeper.UpdateFileChecksum(path, etag, merkleHash);
624 state.Hash = merkleHash;
629 //Does the file exist in the container's local folder?
630 public bool Exists(string relativePath)
632 if (String.IsNullOrWhiteSpace(relativePath))
633 throw new ArgumentNullException("relativePath");
634 //A RootPath must be set before calling this method
635 if (String.IsNullOrWhiteSpace(RootPath))
636 throw new InvalidOperationException("RootPath was not set");
637 Contract.EndContractBlock();
638 //Create the absolute path by combining the RootPath with the relativePath
639 var absolutePath=Path.Combine(RootPath, relativePath);
640 //Is this a valid file?
641 if (File.Exists(absolutePath))
644 if (Directory.Exists(absolutePath))
646 //Fail if it is neither
650 public static FileAgent GetFileAgent(AccountInfo accountInfo)
652 return GetFileAgent(accountInfo.AccountPath);
655 public static FileAgent GetFileAgent(string rootPath)
657 return AgentLocator<FileAgent>.Get(rootPath.ToLower());
661 public FileSystemInfo GetFileSystemInfo(string relativePath)
663 if (String.IsNullOrWhiteSpace(relativePath))
664 throw new ArgumentNullException("relativePath");
665 //A RootPath must be set before calling this method
666 if (String.IsNullOrWhiteSpace(RootPath))
667 throw new InvalidOperationException("RootPath was not set");
668 Contract.EndContractBlock();
670 var absolutePath = Path.Combine(RootPath, relativePath);
672 if (Directory.Exists(absolutePath))
673 return new DirectoryInfo(absolutePath).WithProperCapitalization();
675 return new FileInfo(absolutePath).WithProperCapitalization();
679 public void Delete(string relativePath)
681 var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
682 if (Log.IsDebugEnabled)
683 Log.DebugFormat("Deleting {0}", absolutePath);
684 if (File.Exists(absolutePath))
688 File.Delete(absolutePath);
690 //The file may have been deleted by another thread. Just ignore the relevant exception
691 catch (FileNotFoundException) { }
693 else if (Directory.Exists(absolutePath))
698 var dirinfo = new DirectoryInfo(absolutePath);
699 dirinfo.Attributes &= ~FileAttributes.ReadOnly;
700 foreach (var info in dirinfo.EnumerateFileSystemInfos("*",SearchOption.AllDirectories))
702 info.Attributes &= ~FileAttributes.ReadOnly;
705 dirinfo.Delete(true);
707 //The directory may have been deleted by another thread. Just ignore the relevant exception
708 catch (DirectoryNotFoundException){}
711 //_ignoreFiles[absolutePath] = absolutePath;
712 StatusKeeper.ClearFileStatus(absolutePath);