code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/FileAgent.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.ComponentModel.Composition;
   4 using System.Diagnostics;
   5 using System.Diagnostics.Contracts;
   6 using System.IO;
   7 using System.Linq;
   8 using System.Text;
   9 using System.Threading.Tasks;
  10 using Pithos.Interfaces;
  11 using Pithos.Network;
  12 using log4net;
  13 using log4net.Core;
  14
  15 namespace Pithos.Core.Agents
  16 {
  17 //    [Export]
  18     public class FileAgent
  19     {
  20         Agent<WorkflowState> _agent;
  21         private FileSystemWatcher _watcher;
  22
  23         //[Import]
  24         public IStatusKeeper StatusKeeper { get; set; }
  25         //[Import]
  26         public IPithosWorkflow Workflow { get; set; }
  27         //[Import]
  28         public WorkflowAgent WorkflowAgent { get; set; }
  29
  30         private AccountInfo AccountInfo { get; set; }
  31
  32         private string RootPath { get;  set; }
  33
  34         private static readonly ILog Log = LogManager.GetLogger("FileAgent");
  35
  36         public void Start(AccountInfo accountInfo,string rootPath)
  37         {
  38             if (accountInfo==null)
  39                 throw new ArgumentNullException("accountInfo");
  40             if (String.IsNullOrWhiteSpace(rootPath))
  41                 throw new ArgumentNullException("rootPath");
  42             if (!Path.IsPathRooted(rootPath))
  43                 throw new ArgumentException("rootPath must be an absolute path","rootPath");
  44             Contract.EndContractBlock();
  45
  46             AccountInfo = accountInfo;
  47             RootPath = rootPath;
  48             _watcher = new FileSystemWatcher(rootPath);
  49             _watcher.IncludeSubdirectories = true;
  50             _watcher.Changed += OnFileEvent;
  51             _watcher.Created += OnFileEvent;
  52             _watcher.Deleted += OnFileEvent;
  53             _watcher.Renamed += OnRenameEvent;
  54             _watcher.EnableRaisingEvents = true;
  55
  56
  57             _agent = Agent<WorkflowState>.Start(inbox =>
  58             {
  59                 Action loop = null;
  60                 loop = () =>
  61                 {
  62                     var message = inbox.Receive();
  63                     var process=message.Then(Process,inbox.CancellationToken);
  64
  65                     inbox.LoopAsync(process,loop,ex=>
  66                         Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
  67                 };
  68                 loop();
  69             });
  70         }
  71
  72         private Task<object> Process(WorkflowState state)
  73         {
  74             if (state==null)
  75                 throw new ArgumentNullException("state");
  76             Contract.EndContractBlock();
  77
  78             Debug.Assert(!Ignore(state.Path));
  79
  80             var networkState = NetworkGate.GetNetworkState(state.Path);
  81             //Skip if the file is already being downloaded or uploaded and
  82             //the change is create or modify
  83             if (networkState != NetworkOperation.None &&
  84                 (
  85                     state.TriggeringChange == WatcherChangeTypes.Created ||
  86                     state.TriggeringChange == WatcherChangeTypes.Changed
  87                 ))
  88                 return CompletedTask<object>.Default;
  89
  90             try
  91             {
  92                 UpdateFileStatus(state);
  93                 UpdateOverlayStatus(state);
  94                 UpdateFileChecksum(state);
  95                 WorkflowAgent.Post(state);
  96             }
  97             catch (IOException exc)
  98             {
  99                 if (File.Exists(state.Path))
 100                 {
 101                     Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
 102                     _agent.Post(state);
 103                 }
 104                 else
 105                 {
 106                     Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
 107                 }
 108             }
 109             catch (Exception exc)
 110             {
 111                 Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
 112                                state.Path, exc);
 113             }
 114             return CompletedTask<object>.Default;
 115         }
 116
 117         public bool Pause
 118         {
 119             get { return _watcher == null || !_watcher.EnableRaisingEvents; }
 120             set
 121             {
 122                 if (_watcher != null)
 123                     _watcher.EnableRaisingEvents = !value;
 124             }
 125         }
 126
 127         public string CachePath { get; set; }
 128
 129         private List<string> _selectivePaths = new List<string>();
 130         public List<string> SelectivePaths
 131         {
 132             get { return _selectivePaths; }
 133             set { _selectivePaths = value; }
 134         }
 135
 136
 137         public void Post(WorkflowState workflowState)
 138         {
 139             if (workflowState == null)
 140                 throw new ArgumentNullException("workflowState");
 141             Contract.EndContractBlock();
 142
 143             _agent.Post(workflowState);
 144         }
 145
 146         public void Stop()
 147         {
 148             if (_watcher != null)
 149             {
 150                 _watcher.Changed -= OnFileEvent;
 151                 _watcher.Created -= OnFileEvent;
 152                 _watcher.Deleted -= OnFileEvent;
 153                 _watcher.Renamed -= OnRenameEvent;
 154                 _watcher.Dispose();
 155             }
 156             _watcher = null;
 157
 158             if (_agent!=null)
 159                 _agent.Stop();
 160         }
 161
 162         // Enumerate all files in the Pithos directory except those in the Fragment folder
 163         // and files with a .ignore extension
 164         public IEnumerable<string> EnumerateFiles(string searchPattern="*")
 165         {
 166             var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
 167                                  where !Ignore(filePath)
 168                                  select filePath;
 169             return monitoredFiles;
 170         }
 171
 172         public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
 173         {
 174             var rootDir = new DirectoryInfo(RootPath);
 175             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 176                                  where !Ignore(file.FullName)
 177                                  select file;
 178             return monitoredFiles;
 179         }
 180
 181         public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
 182         {
 183             var rootDir = new DirectoryInfo(RootPath);
 184             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 185                                  where !Ignore(file.FullName)
 186                                  select file.AsRelativeUrlTo(RootPath);
 187             return monitoredFiles;
 188         }
 189
 190
 191
 192
 193         private bool Ignore(string filePath)
 194         {
 195             if (filePath.StartsWith(CachePath))
 196                 return true;
 197             if (_ignoreFiles.ContainsKey(filePath.ToLower()))
 198                 return true;
 199             return false;
 200         }
 201
 202         //Post a Change message for all events except rename
 203         void OnFileEvent(object sender, FileSystemEventArgs e)
 204         {
 205             //Ignore events that affect the cache folder
 206             var filePath = e.FullPath;
 207             if (Ignore(filePath))
 208                 return;
 209             if (Directory.Exists(filePath))
 210                 return;
 211             _agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
 212         }
 213
 214
 215         //Post a Change message for renames containing the old and new names
 216         void OnRenameEvent(object sender, RenamedEventArgs e)
 217         {
 218             var oldFullPath = e.OldFullPath;
 219             var fullPath = e.FullPath;
 220             if (Ignore(oldFullPath) || Ignore(fullPath))
 221                 return;
 222
 223             _agent.Post(new WorkflowState
 224             {
 225                 AccountInfo=AccountInfo,
 226                 OldPath = oldFullPath,
 227                 OldFileName = e.OldName,
 228                 Path = fullPath,
 229                 FileName = e.Name,
 230                 TriggeringChange = e.ChangeType
 231             });
 232         }
 233
 234
 235
 236         private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
 237         {
 238             {WatcherChangeTypes.Created,FileStatus.Created},
 239             {WatcherChangeTypes.Changed,FileStatus.Modified},
 240             {WatcherChangeTypes.Deleted,FileStatus.Deleted},
 241             {WatcherChangeTypes.Renamed,FileStatus.Renamed}
 242         };
 243
 244         private Dictionary<string,string> _ignoreFiles=new Dictionary<string, string>();
 245
 246         private WorkflowState UpdateFileStatus(WorkflowState state)
 247         {
 248             if (state==null)
 249                 throw new ArgumentNullException("state");
 250             if (String.IsNullOrWhiteSpace(state.Path))
 251                 throw new ArgumentException("The state's Path can't be empty","state");
 252             Contract.EndContractBlock();
 253
 254             var path = state.Path;
 255             var status = _statusDict[state.TriggeringChange];
 256             var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
 257             if (status == oldStatus)
 258             {
 259                 state.Status = status;
 260                 state.Skip = true;
 261                 return state;
 262             }
 263             if (state.Status == FileStatus.Renamed)
 264                 Workflow.ClearFileStatus(path);
 265
 266             state.Status = Workflow.SetFileStatus(path, status);
 267             return state;
 268         }
 269
 270         private WorkflowState UpdateOverlayStatus(WorkflowState state)
 271         {
 272             if (state==null)
 273                 throw new ArgumentNullException("state");
 274             Contract.EndContractBlock();
 275
 276             if (state.Skip)
 277                 return state;
 278
 279             switch (state.Status)
 280             {
 281                 case FileStatus.Created:
 282                 case FileStatus.Modified:
 283                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 284                     break;
 285                 case FileStatus.Deleted:
 286                     //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
 287                     break;
 288                 case FileStatus.Renamed:
 289                     this.StatusKeeper.ClearFileStatus(state.OldPath);
 290                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 291                     break;
 292                 case FileStatus.Unchanged:
 293                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
 294                     break;
 295             }
 296
 297             if (state.Status == FileStatus.Deleted)
 298                 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
 299             else
 300                 NativeMethods.RaiseChangeNotification(state.Path);
 301             return state;
 302         }
 303
 304
 305         private WorkflowState UpdateFileChecksum(WorkflowState state)
 306         {
 307             if (state.Skip)
 308                 return state;
 309
 310             if (state.Status == FileStatus.Deleted)
 311                 return state;
 312
 313             var path = state.Path;
 314             //Skip calculation for folders
 315             if (Directory.Exists(path))
 316                 return state;
 317
 318             var info = new FileInfo(path);
 319             string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
 320             StatusKeeper.UpdateFileChecksum(path, hash);
 321
 322             state.Hash = hash;
 323             return state;
 324         }
 325
 326         //Does the file exist in the container's local folder?
 327         public bool Exists(string relativePath)
 328         {
 329             if (String.IsNullOrWhiteSpace(relativePath))
 330                 throw new ArgumentNullException("relativePath");
 331             //A RootPath must be set before calling this method
 332             if (String.IsNullOrWhiteSpace(RootPath))
 333                 throw new InvalidOperationException("RootPath was not set");
 334             Contract.EndContractBlock();
 335             //Create the absolute path by combining the RootPath with the relativePath
 336             var absolutePath=Path.Combine(RootPath, relativePath);
 337             //Is this a valid file?
 338             if (File.Exists(absolutePath))
 339                 return true;
 340             //Or a directory?
 341             if (Directory.Exists(absolutePath))
 342                 return true;
 343             //Fail if it is neither
 344             return false;
 345         }
 346
 347         public FileInfo GetFileInfo(string relativePath)
 348         {
 349             if (String.IsNullOrWhiteSpace(relativePath))
 350                 throw new ArgumentNullException("relativePath");
 351             //A RootPath must be set before calling this method
 352             if (String.IsNullOrWhiteSpace(RootPath))
 353                 throw new InvalidOperationException("RootPath was not set");
 354             Contract.EndContractBlock();
 355
 356             var absolutePath = Path.Combine(RootPath, relativePath);
 357 //            Debug.Assert(File.Exists(absolutePath),String.Format("Path {0} doesn't exist",absolutePath));
 358
 359             return new FileInfo(absolutePath);
 360
 361         }
 362
 363         public void Delete(string relativePath)
 364         {
 365             var absolutePath = Path.Combine(RootPath, relativePath);
 366             if (File.Exists(absolutePath))
 367             {
 368                 File.Delete(absolutePath);
 369                 _ignoreFiles[absolutePath.ToLower()] = absolutePath.ToLower();
 370             }
 371             StatusKeeper.ClearFileStatus(absolutePath);
 372         }
 373     }
 374 }