code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/FileAgent.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.ComponentModel.Composition;
   4 using System.Diagnostics;
   5 using System.Diagnostics.Contracts;
   6 using System.IO;
   7 using System.Linq;
   8 using System.Text;
   9 using System.Threading.Tasks;
  10 using Pithos.Interfaces;
  11 using Pithos.Network;
  12 using log4net;
  13 using log4net.Core;
  14
  15 namespace Pithos.Core.Agents
  16 {
  17 //    [Export]
  18     public class FileAgent
  19     {
  20         Agent<WorkflowState> _agent;
  21         private FileSystemWatcher _watcher;
  22
  23         //[Import]
  24         public IStatusKeeper StatusKeeper { get; set; }
  25         //[Import]
  26         public IPithosWorkflow Workflow { get; set; }
  27         //[Import]
  28         public WorkflowAgent WorkflowAgent { get; set; }
  29
  30         private AccountInfo AccountInfo { get; set; }
  31
  32         private string RootPath { get;  set; }
  33
  34         private static readonly ILog Log = LogManager.GetLogger("FileAgent");
  35
  36         public void Start(AccountInfo accountInfo,string rootPath)
  37         {
  38             if (accountInfo==null)
  39                 throw new ArgumentNullException("accountInfo");
  40             if (String.IsNullOrWhiteSpace(rootPath))
  41                 throw new ArgumentNullException("rootPath");
  42             if (!Path.IsPathRooted(rootPath))
  43                 throw new ArgumentException("rootPath must be an absolute path","rootPath");
  44             Contract.EndContractBlock();
  45
  46             AccountInfo = accountInfo;
  47             RootPath = rootPath;
  48             _watcher = new FileSystemWatcher(rootPath);
  49             _watcher.IncludeSubdirectories = true;
  50             _watcher.Changed += OnFileEvent;
  51             _watcher.Created += OnFileEvent;
  52             _watcher.Deleted += OnFileEvent;
  53             _watcher.Renamed += OnRenameEvent;
  54             _watcher.EnableRaisingEvents = true;
  55
  56
  57             _agent = Agent<WorkflowState>.Start(inbox =>
  58             {
  59                 Action loop = null;
  60                 loop = () =>
  61                 {
  62                     var message = inbox.Receive();
  63                     var process=message.Then(Process,inbox.CancellationToken);
  64
  65                     inbox.LoopAsync(process,loop,ex=>
  66                         Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
  67                 };
  68                 loop();
  69             });
  70         }
  71
  72         private Task<object> Process(WorkflowState state)
  73         {
  74             if (state==null)
  75                 throw new ArgumentNullException("state");
  76             Contract.EndContractBlock();
  77
  78             if (Ignore(state.Path))
  79                 return CompletedTask<object>.Default;
  80
  81             var networkState = NetworkGate.GetNetworkState(state.Path);
  82             //Skip if the file is already being downloaded or uploaded and
  83             //the change is create or modify
  84             if (networkState != NetworkOperation.None &&
  85                 (
  86                     state.TriggeringChange == WatcherChangeTypes.Created ||
  87                     state.TriggeringChange == WatcherChangeTypes.Changed
  88                 ))
  89                 return CompletedTask<object>.Default;
  90
  91             try
  92             {
  93                 UpdateFileStatus(state);
  94                 UpdateOverlayStatus(state);
  95                 UpdateFileChecksum(state);
  96                 WorkflowAgent.Post(state);
  97             }
  98             catch (IOException exc)
  99             {
 100                 if (File.Exists(state.Path))
 101                 {
 102                     Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
 103                     _agent.Post(state);
 104                 }
 105                 else
 106                 {
 107                     Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
 108                 }
 109             }
 110             catch (Exception exc)
 111             {
 112                 Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
 113                                state.Path, exc);
 114             }
 115             return CompletedTask<object>.Default;
 116         }
 117
 118         public bool Pause
 119         {
 120             get { return _watcher == null || !_watcher.EnableRaisingEvents; }
 121             set
 122             {
 123                 if (_watcher != null)
 124                     _watcher.EnableRaisingEvents = !value;
 125             }
 126         }
 127
 128         public string CachePath { get; set; }
 129
 130         private List<string> _selectivePaths = new List<string>();
 131         public List<string> SelectivePaths
 132         {
 133             get { return _selectivePaths; }
 134             set { _selectivePaths = value; }
 135         }
 136
 137
 138         public void Post(WorkflowState workflowState)
 139         {
 140             if (workflowState == null)
 141                 throw new ArgumentNullException("workflowState");
 142             Contract.EndContractBlock();
 143
 144             _agent.Post(workflowState);
 145         }
 146
 147         public void Stop()
 148         {
 149             if (_watcher != null)
 150             {
 151                 _watcher.Changed -= OnFileEvent;
 152                 _watcher.Created -= OnFileEvent;
 153                 _watcher.Deleted -= OnFileEvent;
 154                 _watcher.Renamed -= OnRenameEvent;
 155                 _watcher.Dispose();
 156             }
 157             _watcher = null;
 158
 159             if (_agent!=null)
 160                 _agent.Stop();
 161         }
 162
 163         // Enumerate all files in the Pithos directory except those in the Fragment folder
 164         // and files with a .ignore extension
 165         public IEnumerable<string> EnumerateFiles(string searchPattern="*")
 166         {
 167             var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
 168                                  where !Ignore(filePath)
 169                                  select filePath;
 170             return monitoredFiles;
 171         }
 172
 173         public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
 174         {
 175             var rootDir = new DirectoryInfo(RootPath);
 176             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 177                                  where !Ignore(file.FullName)
 178                                  select file;
 179             return monitoredFiles;
 180         }
 181
 182         public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
 183         {
 184             var rootDir = new DirectoryInfo(RootPath);
 185             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 186                                  where !Ignore(file.FullName)
 187                                  select file.AsRelativeUrlTo(RootPath);
 188             return monitoredFiles;
 189         }
 190
 191
 192
 193
 194         private bool Ignore(string filePath)
 195         {
 196             var pithosPath = Path.Combine(RootPath, "pithos");
 197             if (pithosPath.Equals(filePath, StringComparison.InvariantCultureIgnoreCase))
 198                 return true;
 199             if (filePath.StartsWith(CachePath))
 200                 return true;
 201             if (_ignoreFiles.ContainsKey(filePath.ToLower()))
 202                 return true;
 203             return false;
 204         }
 205
 206         //Post a Change message for all events except rename
 207         void OnFileEvent(object sender, FileSystemEventArgs e)
 208         {
 209             //Ignore events that affect the cache folder
 210             var filePath = e.FullPath;
 211             if (Ignore(filePath))
 212                 return;
 213           /*  if (Directory.Exists(filePath))
 214                 return;    */
 215             _agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
 216         }
 217
 218
 219         //Post a Change message for renames containing the old and new names
 220         void OnRenameEvent(object sender, RenamedEventArgs e)
 221         {
 222             var oldFullPath = e.OldFullPath;
 223             var fullPath = e.FullPath;
 224             if (Ignore(oldFullPath) || Ignore(fullPath))
 225                 return;
 226
 227             _agent.Post(new WorkflowState
 228             {
 229                 AccountInfo=AccountInfo,
 230                 OldPath = oldFullPath,
 231                 OldFileName = e.OldName,
 232                 Path = fullPath,
 233                 FileName = e.Name,
 234                 TriggeringChange = e.ChangeType
 235             });
 236         }
 237
 238
 239
 240         private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
 241         {
 242             {WatcherChangeTypes.Created,FileStatus.Created},
 243             {WatcherChangeTypes.Changed,FileStatus.Modified},
 244             {WatcherChangeTypes.Deleted,FileStatus.Deleted},
 245             {WatcherChangeTypes.Renamed,FileStatus.Renamed}
 246         };
 247
 248         private Dictionary<string,string> _ignoreFiles=new Dictionary<string, string>();
 249
 250         private WorkflowState UpdateFileStatus(WorkflowState state)
 251         {
 252             if (state==null)
 253                 throw new ArgumentNullException("state");
 254             if (String.IsNullOrWhiteSpace(state.Path))
 255                 throw new ArgumentException("The state's Path can't be empty","state");
 256             Contract.EndContractBlock();
 257
 258             var path = state.Path;
 259             var status = _statusDict[state.TriggeringChange];
 260             var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
 261             if (status == oldStatus)
 262             {
 263                 state.Status = status;
 264                 state.Skip = true;
 265                 return state;
 266             }
 267             if (state.Status == FileStatus.Renamed)
 268                 Workflow.ClearFileStatus(path);
 269
 270             state.Status = Workflow.SetFileStatus(path, status);
 271             return state;
 272         }
 273
 274         private WorkflowState UpdateOverlayStatus(WorkflowState state)
 275         {
 276             if (state==null)
 277                 throw new ArgumentNullException("state");
 278             Contract.EndContractBlock();
 279
 280             if (state.Skip)
 281                 return state;
 282
 283             switch (state.Status)
 284             {
 285                 case FileStatus.Created:
 286                 case FileStatus.Modified:
 287                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 288                     break;
 289                 case FileStatus.Deleted:
 290                     //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
 291                     break;
 292                 case FileStatus.Renamed:
 293                     this.StatusKeeper.ClearFileStatus(state.OldPath);
 294                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 295                     break;
 296                 case FileStatus.Unchanged:
 297                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
 298                     break;
 299             }
 300
 301             if (state.Status == FileStatus.Deleted)
 302                 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
 303             else
 304                 NativeMethods.RaiseChangeNotification(state.Path);
 305             return state;
 306         }
 307
 308
 309         private WorkflowState UpdateFileChecksum(WorkflowState state)
 310         {
 311             if (state.Skip)
 312                 return state;
 313
 314             if (state.Status == FileStatus.Deleted)
 315                 return state;
 316
 317             var path = state.Path;
 318             //Skip calculation for folders
 319             if (Directory.Exists(path))
 320                 return state;
 321
 322             var info = new FileInfo(path);
 323             string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
 324             StatusKeeper.UpdateFileChecksum(path, hash);
 325
 326             state.Hash = hash;
 327             return state;
 328         }
 329
 330         //Does the file exist in the container's local folder?
 331         public bool Exists(string relativePath)
 332         {
 333             if (String.IsNullOrWhiteSpace(relativePath))
 334                 throw new ArgumentNullException("relativePath");
 335             //A RootPath must be set before calling this method
 336             if (String.IsNullOrWhiteSpace(RootPath))
 337                 throw new InvalidOperationException("RootPath was not set");
 338             Contract.EndContractBlock();
 339             //Create the absolute path by combining the RootPath with the relativePath
 340             var absolutePath=Path.Combine(RootPath, relativePath);
 341             //Is this a valid file?
 342             if (File.Exists(absolutePath))
 343                 return true;
 344             //Or a directory?
 345             if (Directory.Exists(absolutePath))
 346                 return true;
 347             //Fail if it is neither
 348             return false;
 349         }
 350
 351         public FileSystemInfo GetFileSystemInfo(string relativePath)
 352         {
 353             if (String.IsNullOrWhiteSpace(relativePath))
 354                 throw new ArgumentNullException("relativePath");
 355             //A RootPath must be set before calling this method
 356             if (String.IsNullOrWhiteSpace(RootPath))
 357                 throw new InvalidOperationException("RootPath was not set");
 358             Contract.EndContractBlock();
 359
 360             var absolutePath = Path.Combine(RootPath, relativePath);
 361
 362             if (Directory.Exists(absolutePath))
 363                 return new DirectoryInfo(absolutePath).WithProperCapitalization();
 364             else
 365                 return new FileInfo(absolutePath).WithProperCapitalization();
 366
 367         }
 368
 369         public void Delete(string relativePath)
 370         {
 371             var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
 372             if (File.Exists(absolutePath))
 373             {
 374                 try
 375                 {
 376                     File.Delete(absolutePath);
 377                 }
 378                 //The file may have been deleted by another thread. Just ignore the relevant exception
 379                 catch (FileNotFoundException) { }
 380             }
 381             else if (Directory.Exists(absolutePath))
 382             {
 383                 try
 384                 {
 385                     Directory.Delete(absolutePath, true);
 386                 }
 387                 //The directory may have been deleted by another thread. Just ignore the relevant exception
 388                 catch (DirectoryNotFoundException){}
 389             }
 390
 391             //_ignoreFiles[absolutePath] = absolutePath;
 392             StatusKeeper.ClearFileStatus(absolutePath);
 393         }
 394     }
 395 }