code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/FileAgent.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.ComponentModel.Composition;
   4 using System.Diagnostics;
   5 using System.Diagnostics.Contracts;
   6 using System.IO;
   7 using System.Linq;
   8 using System.Text;
   9 using System.Threading.Tasks;
  10 using Pithos.Interfaces;
  11 using Pithos.Network;
  12
  13 namespace Pithos.Core.Agents
  14 {
  15     [Export]
  16     public class FileAgent
  17     {
  18         Agent<WorkflowState> _agent;
  19         private FileSystemWatcher _watcher;
  20
  21         [Import]
  22         public IStatusKeeper StatusKeeper { get; set; }
  23         [Import]
  24         public IPithosWorkflow Workflow { get; set; }
  25         [Import]
  26         public WorkflowAgent WorkflowAgent { get; set; }
  27
  28         public string RootPath { get; private set; }
  29
  30         public void Start(string rootPath)
  31         {
  32             RootPath = rootPath;
  33             _watcher = new FileSystemWatcher(rootPath);
  34             _watcher.Changed += OnFileEvent;
  35             _watcher.Created += OnFileEvent;
  36             _watcher.Deleted += OnFileEvent;
  37             _watcher.Renamed += OnRenameEvent;
  38             _watcher.EnableRaisingEvents = true;
  39
  40
  41             _agent = Agent<WorkflowState>.Start(inbox =>
  42             {
  43                 Action loop = null;
  44                 loop = () =>
  45                 {
  46                     var message = inbox.Receive();
  47                     var process=message.Then(Process,inbox.CancellationToken);
  48
  49                     inbox.LoopAsync(process,loop,ex=>
  50                         Trace.TraceError("[ERROR] File Event Processing:\r{0}", ex));
  51                 };
  52                 loop();
  53             });
  54         }
  55
  56         private Task<object> Process(WorkflowState state)
  57         {
  58             Debug.Assert(!Ignore(state.Path));
  59
  60             var networkState = NetworkGate.GetNetworkState(state.Path);
  61             //Skip if the file is already being downloaded or uploaded and
  62             //the change is create or modify
  63             if (networkState != NetworkOperation.None &&
  64                 (
  65                     state.TriggeringChange == WatcherChangeTypes.Created ||
  66                     state.TriggeringChange == WatcherChangeTypes.Changed
  67                 ))
  68                 return CompletedTask<object>.Default;
  69
  70             try
  71             {
  72                 UpdateFileStatus(state);
  73                 UpdateOverlayStatus(state);
  74                 UpdateFileChecksum(state);
  75                 WorkflowAgent.Post(state);
  76             }
  77             catch (IOException exc)
  78             {
  79                 if (File.Exists(state.Path))
  80                 {
  81                     Trace.TraceWarning("File access error occured, retrying {0}\n{1}", state.Path, exc);
  82                     _agent.Post(state);
  83                 }
  84                 else
  85                 {
  86                     Trace.TraceWarning("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
  87                 }
  88             }
  89             catch (Exception exc)
  90             {
  91                 Trace.TraceWarning("Error occured while indexing{0. The file will be skipped}\n{1}", state.Path, exc);
  92             }
  93             return CompletedTask<object>.Default;
  94         }
  95
  96
  97 /*
  98         private Task Process(Task<WorkflowState> action)
  99         {
 100             return action.ContinueWith(t => Process(t.Result));
 101         }
 102 */
 103
 104
 105         public bool Pause
 106         {
 107             get { return _watcher == null || !_watcher.EnableRaisingEvents; }
 108             set
 109             {
 110                 if (_watcher != null)
 111                     _watcher.EnableRaisingEvents = !value;
 112             }
 113         }
 114
 115         public string FragmentsPath { get; set; }
 116
 117         public void Post(WorkflowState workflowState)
 118         {
 119             _agent.Post(workflowState);
 120         }
 121
 122         public void Stop()
 123         {
 124             if (_watcher != null)
 125             {
 126                 _watcher.Changed -= OnFileEvent;
 127                 _watcher.Created -= OnFileEvent;
 128                 _watcher.Deleted -= OnFileEvent;
 129                 _watcher.Renamed -= OnRenameEvent;
 130                 _watcher.Dispose();
 131             }
 132             _watcher = null;
 133
 134             _agent.Stop();
 135         }
 136
 137         // Enumerate all files in the Pithos directory except those in the Fragment folder
 138         // and files with a .ignore extension
 139         public IEnumerable<string> EnumerateFiles(string searchPattern="*")
 140         {
 141             var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
 142                                  where !Ignore(filePath)
 143                                  select filePath;
 144             return monitoredFiles;
 145         }
 146
 147         public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
 148         {
 149             var rootDir = new DirectoryInfo(RootPath);
 150             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 151                                  where !Ignore(file.FullName)
 152                                  select file;
 153             return monitoredFiles;
 154         }
 155
 156         public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
 157         {
 158             var rootDir = new DirectoryInfo(RootPath);
 159             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 160                                  where !Ignore(file.FullName)
 161                                  select file.AsRelativeUrlTo(RootPath);
 162             return monitoredFiles;
 163         }
 164
 165
 166
 167
 168         private bool Ignore(string filePath)
 169         {
 170             if (filePath.StartsWith(FragmentsPath))
 171                 return true;
 172             if (_ignoreFiles.ContainsKey(filePath.ToLower()))
 173                 return true;
 174             return false;
 175         }
 176
 177         //Post a Change message for all events except rename
 178         void OnFileEvent(object sender, FileSystemEventArgs e)
 179         {
 180             //Ignore events that affect the Fragments folder
 181             var filePath = e.FullPath;
 182             if (Ignore(filePath))
 183                 return;
 184             _agent.Post(new WorkflowState { Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
 185         }
 186
 187
 188         //Post a Change message for renames containing the old and new names
 189         void OnRenameEvent(object sender, RenamedEventArgs e)
 190         {
 191             var oldFullPath = e.OldFullPath;
 192             var fullPath = e.FullPath;
 193             if (Ignore(oldFullPath) || Ignore(fullPath))
 194                 return;
 195
 196             _agent.Post(new WorkflowState
 197             {
 198                 OldPath = oldFullPath,
 199                 OldFileName = e.OldName,
 200                 Path = fullPath,
 201                 FileName = e.Name,
 202                 TriggeringChange = e.ChangeType
 203             });
 204         }
 205
 206
 207
 208         private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
 209         {
 210             {WatcherChangeTypes.Created,FileStatus.Created},
 211             {WatcherChangeTypes.Changed,FileStatus.Modified},
 212             {WatcherChangeTypes.Deleted,FileStatus.Deleted},
 213             {WatcherChangeTypes.Renamed,FileStatus.Renamed}
 214         };
 215
 216         private Dictionary<string,string> _ignoreFiles=new Dictionary<string, string>();
 217
 218         private WorkflowState UpdateFileStatus(WorkflowState state)
 219         {
 220             Debug.Assert(!state.Path.Contains("fragments"));
 221             Debug.Assert(!state.Path.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase));
 222
 223             string path = state.Path;
 224             FileStatus status = _statusDict[state.TriggeringChange];
 225             var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
 226             if (status == oldStatus)
 227             {
 228                 state.Status = status;
 229                 state.Skip = true;
 230                 return state;
 231             }
 232             if (state.Status == FileStatus.Renamed)
 233                 Workflow.ClearFileStatus(path);
 234
 235             state.Status = Workflow.SetFileStatus(path, status);
 236             return state;
 237         }
 238
 239         private WorkflowState UpdateOverlayStatus(WorkflowState state)
 240         {
 241             if (state.Skip)
 242                 return state;
 243
 244             switch (state.Status)
 245             {
 246                 case FileStatus.Created:
 247                 case FileStatus.Modified:
 248                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 249                     break;
 250                 case FileStatus.Deleted:
 251                     //this.StatusKeeper.RemoveFileOverlayStatus(state.Path);
 252                     break;
 253                 case FileStatus.Renamed:
 254                     this.StatusKeeper.RemoveFileOverlayStatus(state.OldPath);
 255                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 256                     break;
 257                 case FileStatus.Unchanged:
 258                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
 259                     break;
 260             }
 261
 262             if (state.Status == FileStatus.Deleted)
 263                 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
 264             else
 265                 NativeMethods.RaiseChangeNotification(state.Path);
 266             return state;
 267         }
 268
 269
 270         private WorkflowState UpdateFileChecksum(WorkflowState state)
 271         {
 272             if (state.Skip)
 273                 return state;
 274
 275             if (state.Status == FileStatus.Deleted)
 276                 return state;
 277
 278             var path = state.Path;
 279             //Skip calculation for folders
 280             if (Directory.Exists(path))
 281                 return state;
 282
 283             var info = new FileInfo(path);
 284             string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
 285             StatusKeeper.UpdateFileChecksum(path, hash);
 286
 287             state.Hash = hash;
 288             return state;
 289         }
 290
 291         //Does the file exist in the container's local folder?
 292         public bool Exists(string relativePath)
 293         {
 294             if (String.IsNullOrWhiteSpace(relativePath))
 295                 throw new ArgumentNullException("relativePath");
 296             //A RootPath must be set before calling this method
 297             if (String.IsNullOrWhiteSpace(RootPath))
 298                 throw new InvalidOperationException("RootPath was not set");
 299             Contract.EndContractBlock();
 300             //Create the absolute path by combining the RootPath with the relativePath
 301             var absolutePath=Path.Combine(RootPath, relativePath);
 302             //Is this a valid file?
 303             if (File.Exists(absolutePath))
 304                 return true;
 305             //Or a directory?
 306             if (Directory.Exists(absolutePath))
 307                 return true;
 308             //Fail if it is neither
 309             return false;
 310         }
 311
 312         public FileInfo GetFileInfo(string relativePath)
 313         {
 314             if (String.IsNullOrWhiteSpace(relativePath))
 315                 throw new ArgumentNullException("relativePath");
 316             //A RootPath must be set before calling this method
 317             if (String.IsNullOrWhiteSpace(RootPath))
 318                 throw new InvalidOperationException("RootPath was not set");
 319             Contract.EndContractBlock();
 320
 321             var absolutePath = Path.Combine(RootPath, relativePath);
 322             Debug.Assert(File.Exists(absolutePath),String.Format("Path {0} doesn't exist",absolutePath));
 323
 324             return new FileInfo(absolutePath);
 325
 326         }
 327
 328         public void Delete(string relativePath)
 329         {
 330             var absolutePath = Path.Combine(RootPath, relativePath);
 331             if (File.Exists(absolutePath))
 332             {
 333                 File.Delete(absolutePath);
 334                 _ignoreFiles[absolutePath.ToLower()] = absolutePath.ToLower();
 335             }
 336             StatusKeeper.ClearFileStatus(absolutePath);
 337         }
 338     }
 339 }