code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/FileAgent.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.ComponentModel.Composition;
   4 using System.Diagnostics;
   5 using System.Diagnostics.Contracts;
   6 using System.IO;
   7 using System.Linq;
   8 using System.Text;
   9 using System.Threading.Tasks;
  10 using Pithos.Interfaces;
  11 using Pithos.Network;
  12
  13 namespace Pithos.Core.Agents
  14 {
  15     [Export]
  16     public class FileAgent
  17     {
  18         Agent<WorkflowState> _agent;
  19         private FileSystemWatcher _watcher;
  20
  21         [Import]
  22         public IStatusKeeper StatusKeeper { get; set; }
  23         [Import]
  24         public IPithosWorkflow Workflow { get; set; }
  25         [Import]
  26         public WorkflowAgent WorkflowAgent { get; set; }
  27
  28         public string RootPath { get; private set; }
  29
  30         public void Start(string rootPath)
  31         {
  32             RootPath = rootPath;
  33             _watcher = new FileSystemWatcher(rootPath);
  34             _watcher.Changed += OnFileEvent;
  35             _watcher.Created += OnFileEvent;
  36             _watcher.Deleted += OnFileEvent;
  37             _watcher.Renamed += OnRenameEvent;
  38             _watcher.EnableRaisingEvents = true;
  39
  40
  41             _agent = Agent<WorkflowState>.Start(inbox =>
  42             {
  43                 Action loop = null;
  44                 loop = () =>
  45                 {
  46                     var message = inbox.Receive();
  47                     var process=message.Then(Process,inbox.CancellationToken);
  48
  49                     inbox.LoopAsync(process,loop,ex=>
  50                         Trace.TraceError("[ERROR] File Event Processing:\r{0}", ex));
  51                 };
  52                 loop();
  53             });
  54         }
  55
  56         private Task<object> Process(WorkflowState state)
  57         {
  58             Debug.Assert(!Ignore(state.Path));
  59
  60             var networkState = NetworkGate.GetNetworkState(state.Path);
  61             //Skip if the file is already being downloaded or uploaded and
  62             //the change is create or modify
  63             if (networkState != NetworkOperation.None &&
  64                 (
  65                     state.TriggeringChange == WatcherChangeTypes.Created ||
  66                     state.TriggeringChange == WatcherChangeTypes.Changed
  67                 ))
  68                 return CompletedTask<object>.Default;
  69
  70             try
  71             {
  72                 UpdateFileStatus(state);
  73                 UpdateOverlayStatus(state);
  74                 UpdateFileChecksum(state);
  75                 WorkflowAgent.Post(state);
  76             }
  77             catch (IOException exc)
  78             {
  79                 Trace.TraceWarning("File access error occured, retrying {0}\n{1}", state.Path, exc);
  80                 _agent.Post(state);
  81             }
  82             catch (Exception exc)
  83             {
  84                 Trace.TraceWarning("Error occured while indexing{0. The file will be skipped}\n{1}", state.Path, exc);
  85             }
  86             return CompletedTask<object>.Default;
  87         }
  88
  89
  90 /*
  91         private Task Process(Task<WorkflowState> action)
  92         {
  93             return action.ContinueWith(t => Process(t.Result));
  94         }
  95 */
  96
  97
  98         public bool Pause
  99         {
 100             get { return _watcher == null || !_watcher.EnableRaisingEvents; }
 101             set
 102             {
 103                 if (_watcher != null)
 104                     _watcher.EnableRaisingEvents = !value;
 105             }
 106         }
 107
 108         public string FragmentsPath { get; set; }
 109
 110         public void Post(WorkflowState workflowState)
 111         {
 112             _agent.Post(workflowState);
 113         }
 114
 115         public void Stop()
 116         {
 117             if (_watcher != null)
 118             {
 119                 _watcher.Changed -= OnFileEvent;
 120                 _watcher.Created -= OnFileEvent;
 121                 _watcher.Deleted -= OnFileEvent;
 122                 _watcher.Renamed -= OnRenameEvent;
 123                 _watcher.Dispose();
 124             }
 125             _watcher = null;
 126
 127             _agent.Stop();
 128         }
 129
 130         // Enumerate all files in the Pithos directory except those in the Fragment folder
 131         // and files with a .ignore extension
 132         public IEnumerable<string> EnumerateFiles(string searchPattern="*")
 133         {
 134             var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
 135                                  where !Ignore(filePath)
 136                                  select filePath;
 137             return monitoredFiles;
 138         }
 139
 140         public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
 141         {
 142             var rootDir = new DirectoryInfo(RootPath);
 143             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 144                                  where !Ignore(file.FullName)
 145                                  select file;
 146             return monitoredFiles;
 147         }
 148
 149         public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
 150         {
 151             var rootDir = new DirectoryInfo(RootPath);
 152             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 153                                  where !Ignore(file.FullName)
 154                                  select file.AsRelativeUrlTo(RootPath);
 155             return monitoredFiles;
 156         }
 157
 158
 159
 160
 161         private bool Ignore(string filePath)
 162         {
 163             if (filePath.StartsWith(FragmentsPath))
 164                 return true;
 165             return false;
 166         }
 167
 168         //Post a Change message for all events except rename
 169         void OnFileEvent(object sender, FileSystemEventArgs e)
 170         {
 171             //Ignore events that affect the Fragments folder
 172             var filePath = e.FullPath;
 173             if (Ignore(filePath))
 174                 return;
 175             _agent.Post(new WorkflowState { Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
 176         }
 177
 178
 179         //Post a Change message for renames containing the old and new names
 180         void OnRenameEvent(object sender, RenamedEventArgs e)
 181         {
 182             var oldFullPath = e.OldFullPath;
 183             var fullPath = e.FullPath;
 184             if (Ignore(oldFullPath) || Ignore(fullPath))
 185                 return;
 186
 187             _agent.Post(new WorkflowState
 188             {
 189                 OldPath = oldFullPath,
 190                 OldFileName = e.OldName,
 191                 Path = fullPath,
 192                 FileName = e.Name,
 193                 TriggeringChange = e.ChangeType
 194             });
 195         }
 196
 197
 198
 199         private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
 200         {
 201             {WatcherChangeTypes.Created,FileStatus.Created},
 202             {WatcherChangeTypes.Changed,FileStatus.Modified},
 203             {WatcherChangeTypes.Deleted,FileStatus.Deleted},
 204             {WatcherChangeTypes.Renamed,FileStatus.Renamed}
 205         };
 206
 207         private WorkflowState UpdateFileStatus(WorkflowState state)
 208         {
 209             Debug.Assert(!state.Path.Contains("fragments"));
 210             Debug.Assert(!state.Path.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase));
 211
 212             string path = state.Path;
 213             FileStatus status = _statusDict[state.TriggeringChange];
 214             var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
 215             if (status == oldStatus)
 216             {
 217                 state.Status = status;
 218                 state.Skip = true;
 219                 return state;
 220             }
 221             if (state.Status == FileStatus.Renamed)
 222                 Workflow.ClearFileStatus(path);
 223
 224             state.Status = Workflow.SetFileStatus(path, status);
 225             return state;
 226         }
 227
 228         private WorkflowState UpdateOverlayStatus(WorkflowState state)
 229         {
 230             if (state.Skip)
 231                 return state;
 232
 233             switch (state.Status)
 234             {
 235                 case FileStatus.Created:
 236                 case FileStatus.Modified:
 237                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 238                     break;
 239                 case FileStatus.Deleted:
 240                     this.StatusKeeper.RemoveFileOverlayStatus(state.Path);
 241                     break;
 242                 case FileStatus.Renamed:
 243                     this.StatusKeeper.RemoveFileOverlayStatus(state.OldPath);
 244                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 245                     break;
 246                 case FileStatus.Unchanged:
 247                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
 248                     break;
 249             }
 250
 251             if (state.Status == FileStatus.Deleted)
 252                 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
 253             else
 254                 NativeMethods.RaiseChangeNotification(state.Path);
 255             return state;
 256         }
 257
 258
 259         private WorkflowState UpdateFileChecksum(WorkflowState state)
 260         {
 261             if (state.Skip)
 262                 return state;
 263
 264             if (state.Status == FileStatus.Deleted)
 265                 return state;
 266
 267             var path = state.Path;
 268             //Skip calculation for folders
 269             if (Directory.Exists(path))
 270                 return state;
 271
 272             string hash = Signature.CalculateMD5(path);
 273
 274             StatusKeeper.UpdateFileChecksum(path, hash);
 275
 276             state.Hash = hash;
 277             return state;
 278         }
 279
 280         //Does the file exist in the container's local folder?
 281         public bool Exists(string relativePath)
 282         {
 283             if (String.IsNullOrWhiteSpace(relativePath))
 284                 throw new ArgumentNullException("relativePath");
 285             //A RootPath must be set before calling this method
 286             if (String.IsNullOrWhiteSpace(RootPath))
 287                 throw new InvalidOperationException("RootPath was not set");
 288             Contract.EndContractBlock();
 289             //Create the absolute path by combining the RootPath with the relativePath
 290             var absolutePath=Path.Combine(RootPath, relativePath);
 291             //Is this a valid file?
 292             if (File.Exists(absolutePath))
 293                 return true;
 294             //Or a directory?
 295             if (Directory.Exists(absolutePath))
 296                 return true;
 297             //Fail if it is neither
 298             return false;
 299         }
 300
 301         public FileInfo GetFileInfo(string relativePath)
 302         {
 303             if (String.IsNullOrWhiteSpace(relativePath))
 304                 throw new ArgumentNullException("relativePath");
 305             //A RootPath must be set before calling this method
 306             if (String.IsNullOrWhiteSpace(RootPath))
 307                 throw new InvalidOperationException("RootPath was not set");
 308             Contract.EndContractBlock();
 309
 310             var absolutePath = Path.Combine(RootPath, relativePath);
 311             Debug.Assert(File.Exists(absolutePath),String.Format("Path {0} doesn't exist",absolutePath));
 312
 313             return new FileInfo(absolutePath);
 314
 315         }
 316
 317     }
 318 }