code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/FileAgent.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.ComponentModel.Composition;
   4 using System.Diagnostics;
   5 using System.Diagnostics.Contracts;
   6 using System.IO;
   7 using System.Linq;
   8 using System.Text;
   9 using System.Threading.Tasks;
  10 using Pithos.Interfaces;
  11 using Pithos.Network;
  12 using log4net;
  13 using log4net.Core;
  14
  15 namespace Pithos.Core.Agents
  16 {
  17 //    [Export]
  18     public class FileAgent
  19     {
  20         Agent<WorkflowState> _agent;
  21         private FileSystemWatcher _watcher;
  22         private FileSystemWatcherAdapter _adapter;
  23
  24         //[Import]
  25         public IStatusKeeper StatusKeeper { get; set; }
  26         //[Import]
  27         public IPithosWorkflow Workflow { get; set; }
  28         //[Import]
  29         public WorkflowAgent WorkflowAgent { get; set; }
  30
  31         private AccountInfo AccountInfo { get; set; }
  32
  33         private string RootPath { get;  set; }
  34
  35         private static readonly ILog Log = LogManager.GetLogger("FileAgent");
  36
  37         public void Start(AccountInfo accountInfo,string rootPath)
  38         {
  39             if (accountInfo==null)
  40                 throw new ArgumentNullException("accountInfo");
  41             if (String.IsNullOrWhiteSpace(rootPath))
  42                 throw new ArgumentNullException("rootPath");
  43             if (!Path.IsPathRooted(rootPath))
  44                 throw new ArgumentException("rootPath must be an absolute path","rootPath");
  45             Contract.EndContractBlock();
  46
  47             AccountInfo = accountInfo;
  48             RootPath = rootPath;
  49             _watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true};
  50             _adapter = new FileSystemWatcherAdapter(_watcher);
  51
  52             _adapter.Changed += OnFileEvent;
  53             _adapter.Created += OnFileEvent;
  54             _adapter.Deleted += OnFileEvent;
  55             _adapter.Renamed += OnRenameEvent;
  56             _adapter.Moved += OnMoveEvent;
  57             _watcher.EnableRaisingEvents = true;
  58
  59
  60             _agent = Agent<WorkflowState>.Start(inbox =>
  61             {
  62                 Action loop = null;
  63                 loop = () =>
  64                 {
  65                     var message = inbox.Receive();
  66                     var process=message.Then(Process,inbox.CancellationToken);
  67
  68                     inbox.LoopAsync(process,loop,ex=>
  69                         Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
  70                 };
  71                 loop();
  72             });
  73         }
  74
  75         private Task<object> Process(WorkflowState state)
  76         {
  77             if (state==null)
  78                 throw new ArgumentNullException("state");
  79             Contract.EndContractBlock();
  80
  81             if (Ignore(state.Path))
  82                 return CompletedTask<object>.Default;
  83
  84             var networkState = NetworkGate.GetNetworkState(state.Path);
  85             //Skip if the file is already being downloaded or uploaded and
  86             //the change is create or modify
  87             if (networkState != NetworkOperation.None &&
  88                 (
  89                     state.TriggeringChange == WatcherChangeTypes.Created ||
  90                     state.TriggeringChange == WatcherChangeTypes.Changed
  91                 ))
  92                 return CompletedTask<object>.Default;
  93
  94             try
  95             {
  96                 UpdateFileStatus(state);
  97                 UpdateOverlayStatus(state);
  98                 UpdateFileChecksum(state);
  99                 WorkflowAgent.Post(state);
 100             }
 101             catch (IOException exc)
 102             {
 103                 if (File.Exists(state.Path))
 104                 {
 105                     Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
 106                     _agent.Post(state);
 107                 }
 108                 else
 109                 {
 110                     Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
 111                 }
 112             }
 113             catch (Exception exc)
 114             {
 115                 Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
 116                                state.Path, exc);
 117             }
 118             return CompletedTask<object>.Default;
 119         }
 120
 121         public bool Pause
 122         {
 123             get { return _watcher == null || !_watcher.EnableRaisingEvents; }
 124             set
 125             {
 126                 if (_watcher != null)
 127                     _watcher.EnableRaisingEvents = !value;
 128             }
 129         }
 130
 131         public string CachePath { get; set; }
 132
 133         private List<string> _selectivePaths = new List<string>();
 134         public List<string> SelectivePaths
 135         {
 136             get { return _selectivePaths; }
 137             set { _selectivePaths = value; }
 138         }
 139
 140
 141         public void Post(WorkflowState workflowState)
 142         {
 143             if (workflowState == null)
 144                 throw new ArgumentNullException("workflowState");
 145             Contract.EndContractBlock();
 146
 147             _agent.Post(workflowState);
 148         }
 149
 150         public void Stop()
 151         {
 152             if (_watcher != null)
 153             {
 154                 _watcher.Changed -= OnFileEvent;
 155                 _watcher.Created -= OnFileEvent;
 156                 _watcher.Deleted -= OnFileEvent;
 157                 _watcher.Renamed -= OnRenameEvent;
 158                 _watcher.Dispose();
 159             }
 160             _watcher = null;
 161
 162             if (_agent!=null)
 163                 _agent.Stop();
 164         }
 165
 166         // Enumerate all files in the Pithos directory except those in the Fragment folder
 167         // and files with a .ignore extension
 168         public IEnumerable<string> EnumerateFiles(string searchPattern="*")
 169         {
 170             var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
 171                                  where !Ignore(filePath)
 172                                  select filePath;
 173             return monitoredFiles;
 174         }
 175
 176         public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
 177         {
 178             var rootDir = new DirectoryInfo(RootPath);
 179             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 180                                  where !Ignore(file.FullName)
 181                                  select file;
 182             return monitoredFiles;
 183         }
 184
 185         public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
 186         {
 187             var rootDir = new DirectoryInfo(RootPath);
 188             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 189                                  where !Ignore(file.FullName)
 190                                  select file.AsRelativeUrlTo(RootPath);
 191             return monitoredFiles;
 192         }
 193
 194         public IEnumerable<string> EnumerateFilesSystemInfosAsRelativeUrls(string searchPattern="*")
 195         {
 196             var rootDir = new DirectoryInfo(RootPath);
 197             var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories)
 198                                  where !Ignore(file.FullName)
 199                                  select file.AsRelativeUrlTo(RootPath);
 200             return monitoredFiles;
 201         }
 202
 203
 204
 205
 206         private bool Ignore(string filePath)
 207         {
 208             //Ignore all first-level directories and files
 209             if (FoundBelowRoot(filePath, RootPath,1))
 210                 return true;
 211
 212             //Ignore first-level items under the "others" folder.
 213             var othersPath = Path.Combine(RootPath, FolderConstants.OthersFolder);
 214             if (FoundBelowRoot(filePath, othersPath,1))
 215                 return true;
 216
 217             //Ignore second-level (container) folders under the "others" folder.
 218             if (FoundBelowRoot(filePath, othersPath,2))
 219                 return true;
 220
 221
 222             //Ignore anything happening in the cache path
 223             if (filePath.StartsWith(CachePath))
 224                 return true;
 225             if (_ignoreFiles.ContainsKey(filePath.ToLower()))
 226                 return true;
 227             return false;
 228         }
 229
 230 /*        private static bool FoundInRoot(string filePath, string rootPath)
 231         {
 232             //var rootDirectory = new DirectoryInfo(rootPath);
 233
 234             //If the paths are equal, return true
 235             if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
 236                 return true;
 237
 238             //If the filepath is below the root path
 239             if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
 240             {
 241                 //Get the relative path
 242                 var relativePath = filePath.Substring(rootPath.Length + 1);
 243                 //If the relativePath does NOT contains a path separator, we found a match
 244                 return (!relativePath.Contains(@"\"));
 245             }
 246
 247             //If the filepath is not under the root path, return false
 248             return false;
 249         }*/
 250
 251
 252         private static bool FoundBelowRoot(string filePath, string rootPath,int level)
 253         {
 254             //var rootDirectory = new DirectoryInfo(rootPath);
 255
 256             //If the paths are equal, return true
 257             if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
 258                 return true;
 259
 260             //If the filepath is below the root path
 261             if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
 262             {
 263                 //Get the relative path
 264                 var relativePath = filePath.Substring(rootPath.Length + 1);
 265                 //If the relativePath does NOT contains a path separator, we found a match
 266                 var levels=relativePath.ToCharArray().Count(c=>c=='\\')+1;
 267                 return levels==level;
 268             }
 269
 270             //If the filepath is not under the root path, return false
 271             return false;
 272         }
 273
 274         //Post a Change message for all events except rename
 275         void OnFileEvent(object sender, FileSystemEventArgs e)
 276         {
 277             //Ignore events that affect the cache folder
 278             var filePath = e.FullPath;
 279             if (Ignore(filePath))
 280                 return;
 281
 282             _agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
 283         }
 284
 285
 286         //Post a Change message for renames containing the old and new names
 287         void OnRenameEvent(object sender, RenamedEventArgs e)
 288         {
 289             var oldFullPath = e.OldFullPath;
 290             var fullPath = e.FullPath;
 291             if (Ignore(oldFullPath) || Ignore(fullPath))
 292                 return;
 293
 294             _agent.Post(new WorkflowState
 295             {
 296                 AccountInfo=AccountInfo,
 297                 OldPath = oldFullPath,
 298                 OldFileName = e.OldName,
 299                 Path = fullPath,
 300                 FileName = e.Name,
 301                 TriggeringChange = e.ChangeType
 302             });
 303         }
 304
 305         //Post a Change message for renames containing the old and new names
 306         void OnMoveEvent(object sender, MovedEventArgs e)
 307         {
 308             var oldFullPath = e.OldFullPath;
 309             var fullPath = e.FullPath;
 310             if (Ignore(oldFullPath) || Ignore(fullPath))
 311                 return;
 312
 313             _agent.Post(new WorkflowState
 314             {
 315                 AccountInfo=AccountInfo,
 316                 OldPath = oldFullPath,
 317                 OldFileName = e.OldName,
 318                 Path = fullPath,
 319                 FileName = e.Name,
 320                 TriggeringChange = e.ChangeType
 321             });
 322         }
 323
 324
 325
 326         private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
 327         {
 328             {WatcherChangeTypes.Created,FileStatus.Created},
 329             {WatcherChangeTypes.Changed,FileStatus.Modified},
 330             {WatcherChangeTypes.Deleted,FileStatus.Deleted},
 331             {WatcherChangeTypes.Renamed,FileStatus.Renamed}
 332         };
 333
 334         private Dictionary<string,string> _ignoreFiles=new Dictionary<string, string>();
 335
 336         private WorkflowState UpdateFileStatus(WorkflowState state)
 337         {
 338             if (state==null)
 339                 throw new ArgumentNullException("state");
 340             if (String.IsNullOrWhiteSpace(state.Path))
 341                 throw new ArgumentException("The state's Path can't be empty","state");
 342             Contract.EndContractBlock();
 343
 344             var path = state.Path;
 345             var status = _statusDict[state.TriggeringChange];
 346             var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
 347             if (status == oldStatus)
 348             {
 349                 state.Status = status;
 350                 state.Skip = true;
 351                 return state;
 352             }
 353             if (state.Status == FileStatus.Renamed)
 354                 Workflow.ClearFileStatus(path);
 355
 356             state.Status = Workflow.SetFileStatus(path, status);
 357             return state;
 358         }
 359
 360         private WorkflowState UpdateOverlayStatus(WorkflowState state)
 361         {
 362             if (state==null)
 363                 throw new ArgumentNullException("state");
 364             Contract.EndContractBlock();
 365
 366             if (state.Skip)
 367                 return state;
 368
 369             switch (state.Status)
 370             {
 371                 case FileStatus.Created:
 372                 case FileStatus.Modified:
 373                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 374                     break;
 375                 case FileStatus.Deleted:
 376                     //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
 377                     break;
 378                 case FileStatus.Renamed:
 379                     this.StatusKeeper.ClearFileStatus(state.OldPath);
 380                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 381                     break;
 382                 case FileStatus.Unchanged:
 383                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
 384                     break;
 385             }
 386
 387             if (state.Status == FileStatus.Deleted)
 388                 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
 389             else
 390                 NativeMethods.RaiseChangeNotification(state.Path);
 391             return state;
 392         }
 393
 394
 395         private WorkflowState UpdateFileChecksum(WorkflowState state)
 396         {
 397             if (state.Skip)
 398                 return state;
 399
 400             if (state.Status == FileStatus.Deleted)
 401                 return state;
 402
 403             var path = state.Path;
 404             //Skip calculation for folders
 405             if (Directory.Exists(path))
 406                 return state;
 407
 408             var info = new FileInfo(path);
 409             string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
 410             StatusKeeper.UpdateFileChecksum(path, hash);
 411
 412             state.Hash = hash;
 413             return state;
 414         }
 415
 416         //Does the file exist in the container's local folder?
 417         public bool Exists(string relativePath)
 418         {
 419             if (String.IsNullOrWhiteSpace(relativePath))
 420                 throw new ArgumentNullException("relativePath");
 421             //A RootPath must be set before calling this method
 422             if (String.IsNullOrWhiteSpace(RootPath))
 423                 throw new InvalidOperationException("RootPath was not set");
 424             Contract.EndContractBlock();
 425             //Create the absolute path by combining the RootPath with the relativePath
 426             var absolutePath=Path.Combine(RootPath, relativePath);
 427             //Is this a valid file?
 428             if (File.Exists(absolutePath))
 429                 return true;
 430             //Or a directory?
 431             if (Directory.Exists(absolutePath))
 432                 return true;
 433             //Fail if it is neither
 434             return false;
 435         }
 436
 437         public FileSystemInfo GetFileSystemInfo(string relativePath)
 438         {
 439             if (String.IsNullOrWhiteSpace(relativePath))
 440                 throw new ArgumentNullException("relativePath");
 441             //A RootPath must be set before calling this method
 442             if (String.IsNullOrWhiteSpace(RootPath))
 443                 throw new InvalidOperationException("RootPath was not set");
 444             Contract.EndContractBlock();
 445
 446             var absolutePath = Path.Combine(RootPath, relativePath);
 447
 448             if (Directory.Exists(absolutePath))
 449                 return new DirectoryInfo(absolutePath).WithProperCapitalization();
 450             else
 451                 return new FileInfo(absolutePath).WithProperCapitalization();
 452
 453         }
 454
 455         public void Delete(string relativePath)
 456         {
 457             var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
 458             if (File.Exists(absolutePath))
 459             {
 460                 try
 461                 {
 462                     File.Delete(absolutePath);
 463                 }
 464                 //The file may have been deleted by another thread. Just ignore the relevant exception
 465                 catch (FileNotFoundException) { }
 466             }
 467             else if (Directory.Exists(absolutePath))
 468             {
 469                 try
 470                 {
 471                     Directory.Delete(absolutePath, true);
 472                 }
 473                 //The directory may have been deleted by another thread. Just ignore the relevant exception
 474                 catch (DirectoryNotFoundException){}
 475             }
 476
 477             //_ignoreFiles[absolutePath] = absolutePath;
 478             StatusKeeper.ClearFileStatus(absolutePath);
 479         }
 480     }
 481 }