code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/FileAgent.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.ComponentModel.Composition;
   4 using System.Diagnostics;
   5 using System.Diagnostics.Contracts;
   6 using System.IO;
   7 using System.Linq;
   8 using System.Text;
   9 using Pithos.Interfaces;
  10 using Pithos.Network;
  11
  12 namespace Pithos.Core.Agents
  13 {
  14     [Export]
  15     public class FileAgent
  16     {
  17         Agent<WorkflowState> _agent;
  18         private FileSystemWatcher _watcher;
  19
  20         [Import]
  21         public IStatusKeeper StatusKeeper { get; set; }
  22         [Import]
  23         public IPithosWorkflow Workflow { get; set; }
  24         [Import]
  25         public WorkflowAgent WorkflowAgent { get; set; }
  26
  27         public string RootPath { get; private set; }
  28
  29         public void Start(string rootPath)
  30         {
  31             RootPath = rootPath;
  32             _watcher = new FileSystemWatcher(rootPath);
  33             _watcher.Changed += OnFileEvent;
  34             _watcher.Created += OnFileEvent;
  35             _watcher.Deleted += OnFileEvent;
  36             _watcher.Renamed += OnRenameEvent;
  37             _watcher.EnableRaisingEvents = true;
  38
  39
  40             _agent = Agent<WorkflowState>.Start(inbox =>
  41             {
  42                 Action loop = null;
  43                 loop = () =>
  44                 {
  45                     var message = inbox.Receive();
  46                     var process = message.ContinueWith(t =>
  47                     {
  48                         var state = t.Result;
  49                         Process(state);
  50                         inbox.DoAsync(loop);
  51                     });
  52
  53                     process.ContinueWith(t =>
  54                     {
  55                         inbox.DoAsync(loop);
  56                         if (t.IsFaulted)
  57                         {
  58                             var ex = t.Exception.InnerException;
  59                             if (ex is OperationCanceledException)
  60                                 inbox.Stop();
  61                             Trace.TraceError("[ERROR] File Event Processing:\r{0}", ex);
  62                         }
  63                     });
  64
  65                 };
  66                 loop();
  67             });
  68         }
  69
  70         public bool Pause
  71         {
  72             get { return _watcher == null || !_watcher.EnableRaisingEvents; }
  73             set
  74             {
  75                 if (_watcher != null)
  76                     _watcher.EnableRaisingEvents = !value;
  77             }
  78         }
  79
  80         public string FragmentsPath { get; set; }
  81
  82         public void Post(WorkflowState workflowState)
  83         {
  84             _agent.Post(workflowState);
  85         }
  86
  87         public void Stop()
  88         {
  89             if (_watcher != null)
  90             {
  91                 _watcher.Changed -= OnFileEvent;
  92                 _watcher.Created -= OnFileEvent;
  93                 _watcher.Deleted -= OnFileEvent;
  94                 _watcher.Renamed -= OnRenameEvent;
  95                 _watcher.Dispose();
  96             }
  97             _watcher = null;
  98
  99             _agent.Stop();
 100         }
 101
 102         // Enumerate all files in the Pithos directory except those in the Fragment folder
 103         // and files with a .ignore extension
 104         public IEnumerable<string> EnumerateFiles(string searchPattern="*")
 105         {
 106             var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
 107                                  where !Ignore(filePath)
 108                                  select filePath;
 109             return monitoredFiles;
 110         }
 111
 112         public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
 113         {
 114             var rootDir = new DirectoryInfo(RootPath);
 115             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 116                                  where !Ignore(file.FullName)
 117                                  select file;
 118             return monitoredFiles;
 119         }
 120
 121         public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
 122         {
 123             var rootDir = new DirectoryInfo(RootPath);
 124             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 125                                  where !Ignore(file.FullName)
 126                                  select file.AsRelativeUrlTo(RootPath);
 127             return monitoredFiles;
 128         }
 129
 130
 131
 132
 133         private bool Ignore(string filePath)
 134         {
 135             if (filePath.StartsWith(FragmentsPath))
 136                 return true;
 137             return false;
 138         }
 139
 140         //Post a Change message for all events except rename
 141         void OnFileEvent(object sender, FileSystemEventArgs e)
 142         {
 143             //Ignore events that affect the Fragments folder
 144             var filePath = e.FullPath;
 145             if (Ignore(filePath))
 146                 return;
 147             _agent.Post(new WorkflowState { Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
 148         }
 149
 150
 151         //Post a Change message for renames containing the old and new names
 152         void OnRenameEvent(object sender, RenamedEventArgs e)
 153         {
 154             var oldFullPath = e.OldFullPath;
 155             var fullPath = e.FullPath;
 156             if (Ignore(oldFullPath) || Ignore(fullPath))
 157                 return;
 158
 159             _agent.Post(new WorkflowState
 160             {
 161                 OldPath = oldFullPath,
 162                 OldFileName = e.OldName,
 163                 Path = fullPath,
 164                 FileName = e.Name,
 165                 TriggeringChange = e.ChangeType
 166             });
 167         }
 168
 169
 170         private void Process(WorkflowState state)
 171         {
 172             Debug.Assert(!Ignore(state.Path));
 173
 174             var networkState = NetworkGate.GetNetworkState(state.Path);
 175             //Skip if the file is already being downloaded or uploaded and
 176             //the change is create or modify
 177             if (networkState != NetworkOperation.None &&
 178                 (
 179                     state.TriggeringChange == WatcherChangeTypes.Created ||
 180                     state.TriggeringChange == WatcherChangeTypes.Changed
 181                 ))
 182                 return;
 183
 184             try
 185             {
 186                 UpdateFileStatus(state);
 187                 UpdateOverlayStatus(state);
 188                 UpdateFileChecksum(state);
 189                 WorkflowAgent.Post(state);
 190             }
 191             catch (IOException exc)
 192             {
 193                 Trace.TraceWarning("File access error occured, retrying {0}\n{1}", state.Path, exc);
 194                 _agent.Post(state);
 195             }
 196             catch (Exception exc)
 197             {
 198                 Trace.TraceWarning("Error occured while indexing{0. The file will be skipped}\n{1}", state.Path, exc);
 199             }
 200         }
 201
 202         private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
 203         {
 204             {WatcherChangeTypes.Created,FileStatus.Created},
 205             {WatcherChangeTypes.Changed,FileStatus.Modified},
 206             {WatcherChangeTypes.Deleted,FileStatus.Deleted},
 207             {WatcherChangeTypes.Renamed,FileStatus.Renamed}
 208         };
 209
 210         private WorkflowState UpdateFileStatus(WorkflowState state)
 211         {
 212             Debug.Assert(!state.Path.Contains("fragments"));
 213             Debug.Assert(!state.Path.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase));
 214
 215             string path = state.Path;
 216             FileStatus status = _statusDict[state.TriggeringChange];
 217             var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
 218             if (status == oldStatus)
 219             {
 220                 state.Status = status;
 221                 state.Skip = true;
 222                 return state;
 223             }
 224             if (state.Status == FileStatus.Renamed)
 225                 Workflow.ClearFileStatus(path);
 226
 227             state.Status = Workflow.SetFileStatus(path, status);
 228             return state;
 229         }
 230
 231         private WorkflowState UpdateOverlayStatus(WorkflowState state)
 232         {
 233             if (state.Skip)
 234                 return state;
 235
 236             switch (state.Status)
 237             {
 238                 case FileStatus.Created:
 239                 case FileStatus.Modified:
 240                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 241                     break;
 242                 case FileStatus.Deleted:
 243                     this.StatusKeeper.RemoveFileOverlayStatus(state.Path);
 244                     break;
 245                 case FileStatus.Renamed:
 246                     this.StatusKeeper.RemoveFileOverlayStatus(state.OldPath);
 247                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 248                     break;
 249                 case FileStatus.Unchanged:
 250                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
 251                     break;
 252             }
 253
 254             if (state.Status == FileStatus.Deleted)
 255                 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
 256             else
 257                 NativeMethods.RaiseChangeNotification(state.Path);
 258             return state;
 259         }
 260
 261
 262         private WorkflowState UpdateFileChecksum(WorkflowState state)
 263         {
 264             if (state.Skip)
 265                 return state;
 266
 267             if (state.Status == FileStatus.Deleted)
 268                 return state;
 269
 270             var path = state.Path;
 271             //Skip calculation for folders
 272             if (Directory.Exists(path))
 273                 return state;
 274
 275             string hash = Signature.CalculateMD5(path);
 276
 277             StatusKeeper.UpdateFileChecksum(path, hash);
 278
 279             state.Hash = hash;
 280             return state;
 281         }
 282
 283         //Does the file exist in the container's local folder?
 284         public bool Exists(string relativePath)
 285         {
 286             if (String.IsNullOrWhiteSpace(relativePath))
 287                 throw new ArgumentNullException("relativePath");
 288             //A RootPath must be set before calling this method
 289             if (String.IsNullOrWhiteSpace(RootPath))
 290                 throw new InvalidOperationException("RootPath was not set");
 291             Contract.EndContractBlock();
 292             //Create the absolute path by combining the RootPath with the relativePath
 293             var absolutePath=Path.Combine(RootPath, relativePath);
 294             //Is this a valid file?
 295             if (File.Exists(absolutePath))
 296                 return true;
 297             //Or a directory?
 298             if (Directory.Exists(RootPath))
 299                 return true;
 300             //Fail if it is neither
 301             return false;
 302         }
 303
 304         public FileInfo GetFileInfo(string relativePath)
 305         {
 306             if (String.IsNullOrWhiteSpace(relativePath))
 307                 throw new ArgumentNullException("relativePath");
 308             //A RootPath must be set before calling this method
 309             if (String.IsNullOrWhiteSpace(RootPath))
 310                 throw new InvalidOperationException("RootPath was not set");
 311             Contract.EndContractBlock();
 312
 313             var absolutePath = Path.Combine(RootPath, relativePath);
 314             Debug.Assert(File.Exists(absolutePath));
 315
 316             return new FileInfo(absolutePath);
 317
 318         }
 319     }
 320 }