code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/FileAgent.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.ComponentModel.Composition;
   4 using System.Diagnostics;
   5 using System.Diagnostics.Contracts;
   6 using System.IO;
   7 using System.Linq;
   8 using System.Text;
   9 using System.Threading.Tasks;
  10 using Pithos.Interfaces;
  11 using Pithos.Network;
  12 using log4net;
  13 using log4net.Core;
  14
  15 namespace Pithos.Core.Agents
  16 {
  17 //    [Export]
  18     public class FileAgent
  19     {
  20         Agent<WorkflowState> _agent;
  21         private FileSystemWatcher _watcher;
  22
  23         //[Import]
  24         public IStatusKeeper StatusKeeper { get; set; }
  25         //[Import]
  26         public IPithosWorkflow Workflow { get; set; }
  27         //[Import]
  28         public WorkflowAgent WorkflowAgent { get; set; }
  29
  30         private AccountInfo AccountInfo { get; set; }
  31
  32         private string RootPath { get;  set; }
  33
  34         private static readonly ILog Log = LogManager.GetLogger("FileAgent");
  35
  36         public void Start(AccountInfo accountInfo,string rootPath)
  37         {
  38             if (accountInfo==null)
  39                 throw new ArgumentNullException("accountInfo");
  40             if (String.IsNullOrWhiteSpace(rootPath))
  41                 throw new ArgumentNullException("rootPath");
  42             if (!Path.IsPathRooted(rootPath))
  43                 throw new ArgumentException("rootPath must be an absolute path","rootPath");
  44             Contract.EndContractBlock();
  45
  46             AccountInfo = accountInfo;
  47             RootPath = rootPath;
  48             _watcher = new FileSystemWatcher(rootPath);
  49             _watcher.IncludeSubdirectories = true;
  50             _watcher.Changed += OnFileEvent;
  51             _watcher.Created += OnFileEvent;
  52             _watcher.Deleted += OnFileEvent;
  53             _watcher.Renamed += OnRenameEvent;
  54             _watcher.EnableRaisingEvents = true;
  55
  56
  57             _agent = Agent<WorkflowState>.Start(inbox =>
  58             {
  59                 Action loop = null;
  60                 loop = () =>
  61                 {
  62                     var message = inbox.Receive();
  63                     var process=message.Then(Process,inbox.CancellationToken);
  64
  65                     inbox.LoopAsync(process,loop,ex=>
  66                         Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
  67                 };
  68                 loop();
  69             });
  70         }
  71
  72         private Task<object> Process(WorkflowState state)
  73         {
  74             if (state==null)
  75                 throw new ArgumentNullException("state");
  76             Contract.EndContractBlock();
  77
  78             if (Ignore(state.Path))
  79                 return CompletedTask<object>.Default;
  80
  81             var networkState = NetworkGate.GetNetworkState(state.Path);
  82             //Skip if the file is already being downloaded or uploaded and
  83             //the change is create or modify
  84             if (networkState != NetworkOperation.None &&
  85                 (
  86                     state.TriggeringChange == WatcherChangeTypes.Created ||
  87                     state.TriggeringChange == WatcherChangeTypes.Changed
  88                 ))
  89                 return CompletedTask<object>.Default;
  90
  91             try
  92             {
  93                 UpdateFileStatus(state);
  94                 UpdateOverlayStatus(state);
  95                 UpdateFileChecksum(state);
  96                 WorkflowAgent.Post(state);
  97             }
  98             catch (IOException exc)
  99             {
 100                 if (File.Exists(state.Path))
 101                 {
 102                     Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
 103                     _agent.Post(state);
 104                 }
 105                 else
 106                 {
 107                     Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
 108                 }
 109             }
 110             catch (Exception exc)
 111             {
 112                 Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
 113                                state.Path, exc);
 114             }
 115             return CompletedTask<object>.Default;
 116         }
 117
 118         public bool Pause
 119         {
 120             get { return _watcher == null || !_watcher.EnableRaisingEvents; }
 121             set
 122             {
 123                 if (_watcher != null)
 124                     _watcher.EnableRaisingEvents = !value;
 125             }
 126         }
 127
 128         public string CachePath { get; set; }
 129
 130         private List<string> _selectivePaths = new List<string>();
 131         public List<string> SelectivePaths
 132         {
 133             get { return _selectivePaths; }
 134             set { _selectivePaths = value; }
 135         }
 136
 137
 138         public void Post(WorkflowState workflowState)
 139         {
 140             if (workflowState == null)
 141                 throw new ArgumentNullException("workflowState");
 142             Contract.EndContractBlock();
 143
 144             _agent.Post(workflowState);
 145         }
 146
 147         public void Stop()
 148         {
 149             if (_watcher != null)
 150             {
 151                 _watcher.Changed -= OnFileEvent;
 152                 _watcher.Created -= OnFileEvent;
 153                 _watcher.Deleted -= OnFileEvent;
 154                 _watcher.Renamed -= OnRenameEvent;
 155                 _watcher.Dispose();
 156             }
 157             _watcher = null;
 158
 159             if (_agent!=null)
 160                 _agent.Stop();
 161         }
 162
 163         // Enumerate all files in the Pithos directory except those in the Fragment folder
 164         // and files with a .ignore extension
 165         public IEnumerable<string> EnumerateFiles(string searchPattern="*")
 166         {
 167             var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
 168                                  where !Ignore(filePath)
 169                                  select filePath;
 170             return monitoredFiles;
 171         }
 172
 173         public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
 174         {
 175             var rootDir = new DirectoryInfo(RootPath);
 176             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 177                                  where !Ignore(file.FullName)
 178                                  select file;
 179             return monitoredFiles;
 180         }
 181
 182         public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
 183         {
 184             var rootDir = new DirectoryInfo(RootPath);
 185             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 186                                  where !Ignore(file.FullName)
 187                                  select file.AsRelativeUrlTo(RootPath);
 188             return monitoredFiles;
 189         }
 190
 191
 192
 193
 194         private bool Ignore(string filePath)
 195         {
 196             var pithosPath = Path.Combine(RootPath, "pithos");
 197             if (pithosPath.Equals(filePath, StringComparison.InvariantCultureIgnoreCase))
 198                 return true;
 199             if (filePath.StartsWith(CachePath))
 200                 return true;
 201             if (_ignoreFiles.ContainsKey(filePath.ToLower()))
 202                 return true;
 203             return false;
 204         }
 205
 206         //Post a Change message for all events except rename
 207         void OnFileEvent(object sender, FileSystemEventArgs e)
 208         {
 209             //Ignore events that affect the cache folder
 210             var filePath = e.FullPath;
 211             if (Ignore(filePath))
 212                 return;
 213
 214             _agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
 215         }
 216
 217
 218         //Post a Change message for renames containing the old and new names
 219         void OnRenameEvent(object sender, RenamedEventArgs e)
 220         {
 221             var oldFullPath = e.OldFullPath;
 222             var fullPath = e.FullPath;
 223             if (Ignore(oldFullPath) || Ignore(fullPath))
 224                 return;
 225
 226             _agent.Post(new WorkflowState
 227             {
 228                 AccountInfo=AccountInfo,
 229                 OldPath = oldFullPath,
 230                 OldFileName = e.OldName,
 231                 Path = fullPath,
 232                 FileName = e.Name,
 233                 TriggeringChange = e.ChangeType
 234             });
 235         }
 236
 237
 238
 239         private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
 240         {
 241             {WatcherChangeTypes.Created,FileStatus.Created},
 242             {WatcherChangeTypes.Changed,FileStatus.Modified},
 243             {WatcherChangeTypes.Deleted,FileStatus.Deleted},
 244             {WatcherChangeTypes.Renamed,FileStatus.Renamed}
 245         };
 246
 247         private Dictionary<string,string> _ignoreFiles=new Dictionary<string, string>();
 248
 249         private WorkflowState UpdateFileStatus(WorkflowState state)
 250         {
 251             if (state==null)
 252                 throw new ArgumentNullException("state");
 253             if (String.IsNullOrWhiteSpace(state.Path))
 254                 throw new ArgumentException("The state's Path can't be empty","state");
 255             Contract.EndContractBlock();
 256
 257             var path = state.Path;
 258             var status = _statusDict[state.TriggeringChange];
 259             var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
 260             if (status == oldStatus)
 261             {
 262                 state.Status = status;
 263                 state.Skip = true;
 264                 return state;
 265             }
 266             if (state.Status == FileStatus.Renamed)
 267                 Workflow.ClearFileStatus(path);
 268
 269             state.Status = Workflow.SetFileStatus(path, status);
 270             return state;
 271         }
 272
 273         private WorkflowState UpdateOverlayStatus(WorkflowState state)
 274         {
 275             if (state==null)
 276                 throw new ArgumentNullException("state");
 277             Contract.EndContractBlock();
 278
 279             if (state.Skip)
 280                 return state;
 281
 282             switch (state.Status)
 283             {
 284                 case FileStatus.Created:
 285                 case FileStatus.Modified:
 286                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 287                     break;
 288                 case FileStatus.Deleted:
 289                     //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
 290                     break;
 291                 case FileStatus.Renamed:
 292                     this.StatusKeeper.ClearFileStatus(state.OldPath);
 293                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 294                     break;
 295                 case FileStatus.Unchanged:
 296                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
 297                     break;
 298             }
 299
 300             if (state.Status == FileStatus.Deleted)
 301                 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
 302             else
 303                 NativeMethods.RaiseChangeNotification(state.Path);
 304             return state;
 305         }
 306
 307
 308         private WorkflowState UpdateFileChecksum(WorkflowState state)
 309         {
 310             if (state.Skip)
 311                 return state;
 312
 313             if (state.Status == FileStatus.Deleted)
 314                 return state;
 315
 316             var path = state.Path;
 317             //Skip calculation for folders
 318             if (Directory.Exists(path))
 319                 return state;
 320
 321             var info = new FileInfo(path);
 322             string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
 323             StatusKeeper.UpdateFileChecksum(path, hash);
 324
 325             state.Hash = hash;
 326             return state;
 327         }
 328
 329         //Does the file exist in the container's local folder?
 330         public bool Exists(string relativePath)
 331         {
 332             if (String.IsNullOrWhiteSpace(relativePath))
 333                 throw new ArgumentNullException("relativePath");
 334             //A RootPath must be set before calling this method
 335             if (String.IsNullOrWhiteSpace(RootPath))
 336                 throw new InvalidOperationException("RootPath was not set");
 337             Contract.EndContractBlock();
 338             //Create the absolute path by combining the RootPath with the relativePath
 339             var absolutePath=Path.Combine(RootPath, relativePath);
 340             //Is this a valid file?
 341             if (File.Exists(absolutePath))
 342                 return true;
 343             //Or a directory?
 344             if (Directory.Exists(absolutePath))
 345                 return true;
 346             //Fail if it is neither
 347             return false;
 348         }
 349
 350         public FileSystemInfo GetFileSystemInfo(string relativePath)
 351         {
 352             if (String.IsNullOrWhiteSpace(relativePath))
 353                 throw new ArgumentNullException("relativePath");
 354             //A RootPath must be set before calling this method
 355             if (String.IsNullOrWhiteSpace(RootPath))
 356                 throw new InvalidOperationException("RootPath was not set");
 357             Contract.EndContractBlock();
 358
 359             var absolutePath = Path.Combine(RootPath, relativePath);
 360
 361             if (Directory.Exists(absolutePath))
 362                 return new DirectoryInfo(absolutePath).WithProperCapitalization();
 363             else
 364                 return new FileInfo(absolutePath).WithProperCapitalization();
 365
 366         }
 367
 368         public void Delete(string relativePath)
 369         {
 370             var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
 371             if (File.Exists(absolutePath))
 372             {
 373                 try
 374                 {
 375                     File.Delete(absolutePath);
 376                 }
 377                 //The file may have been deleted by another thread. Just ignore the relevant exception
 378                 catch (FileNotFoundException) { }
 379             }
 380             else if (Directory.Exists(absolutePath))
 381             {
 382                 try
 383                 {
 384                     Directory.Delete(absolutePath, true);
 385                 }
 386                 //The directory may have been deleted by another thread. Just ignore the relevant exception
 387                 catch (DirectoryNotFoundException){}
 388             }
 389
 390             //_ignoreFiles[absolutePath] = absolutePath;
 391             StatusKeeper.ClearFileStatus(absolutePath);
 392         }
 393     }
 394 }