code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/FileAgent.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.ComponentModel.Composition;
   4 using System.Diagnostics;
   5 using System.Diagnostics.Contracts;
   6 using System.IO;
   7 using System.Linq;
   8 using System.Text;
   9 using System.Threading.Tasks;
  10 using System.Threading.Tasks.Dataflow;
  11 using Pithos.Interfaces;
  12 using Pithos.Network;
  13 using log4net;
  14 using log4net.Core;
  15
  16 namespace Pithos.Core.Agents
  17 {
  18     [Export]
  19     public class FileAgent
  20     {
  21         //Agent<WorkflowState> _agent;
  22         TransformBlock<WorkflowState, WorkflowState> _agent;
  23         private FileSystemWatcher _watcher;
  24
  25         [Import]
  26         public IStatusKeeper StatusKeeper { get; set; }
  27         [Import]
  28         public IPithosWorkflow Workflow { get; set; }
  29         [Import]
  30         public WorkflowAgent WorkflowAgent { get; set; }
  31
  32         private AccountInfo AccountInfo { get; set; }
  33
  34         private string RootPath { get;  set; }
  35
  36         private static readonly ILog Log = LogManager.GetLogger("FileAgent");
  37
  38         public FileAgent()
  39         {
  40             _agent = new TransformBlock<WorkflowState, WorkflowState>(async message =>
  41                 await TaskEx.Run(()=>Process(message)));
  42         }
  43
  44         private WorkflowState Process(WorkflowState message)
  45         {
  46             Debug.Assert(!Ignore(message.Path));
  47
  48             try
  49             {
  50                 //Skip if the file is already being downloaded or uploaded and
  51                 //the change is create or modify
  52                 var networkState = NetworkGate.GetNetworkState(message.Path);
  53                 if (networkState != NetworkOperation.None &&
  54                     (message.TriggeringChange == WatcherChangeTypes.Created ||
  55                      message.TriggeringChange == WatcherChangeTypes.Changed))
  56                     return null;
  57
  58                 UpdateFileStatus(message);
  59                 UpdateOverlayStatus(message);
  60                 UpdateFileChecksum(message);
  61                 return message;
  62             }
  63             catch (IOException exc)
  64             {
  65                 if (File.Exists(message.Path))
  66                 {
  67                     Log.WarnFormat("File access error occured, retrying {0}\n{1}", message.Path, exc);
  68                     _agent.Post(message);
  69                 }
  70                 else
  71                 {
  72                     Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", message.Path, exc);
  73                 }
  74             }
  75             catch (Exception exc)
  76             {
  77                 Log.WarnFormat("Error occured while indexing{0}. The file will be skipped}\n{1}",
  78                                message.Path, exc);
  79             }
  80
  81             return null;
  82         }
  83
  84         public void Start(AccountInfo accountInfo,string rootPath)
  85         {
  86             if (accountInfo==null)
  87                 throw new ArgumentNullException("accountInfo");
  88             if (String.IsNullOrWhiteSpace(rootPath))
  89                 throw new ArgumentNullException("rootPath");
  90             if (!Path.IsPathRooted(rootPath))
  91                 throw new ArgumentException("rootPath must be an absolute path","rootPath");
  92             Contract.EndContractBlock();
  93
  94             _agent.LinkTo(WorkflowAgent.Agent);
  95
  96             AccountInfo = accountInfo;
  97             RootPath = rootPath;
  98             _watcher = new FileSystemWatcher(rootPath);
  99             _watcher.IncludeSubdirectories = true;
 100             _watcher.Changed += OnFileEvent;
 101             _watcher.Created += OnFileEvent;
 102             _watcher.Deleted += OnFileEvent;
 103             _watcher.Renamed += OnRenameEvent;
 104             _watcher.EnableRaisingEvents = true;
 105
 106         }
 107
 108         public bool Pause
 109         {
 110             get { return _watcher == null || !_watcher.EnableRaisingEvents; }
 111             set
 112             {
 113                 if (_watcher != null)
 114                     _watcher.EnableRaisingEvents = !value;
 115             }
 116         }
 117
 118         public string CachePath { get; set; }
 119
 120         private List<string> _selectivePaths = new List<string>();
 121         public List<string> SelectivePaths
 122         {
 123             get { return _selectivePaths; }
 124             set { _selectivePaths = value; }
 125         }
 126
 127
 128         public void Post(WorkflowState workflowState)
 129         {
 130             if (workflowState == null)
 131                 throw new ArgumentNullException("workflowState");
 132             Contract.EndContractBlock();
 133
 134             _agent.Post(workflowState);
 135         }
 136
 137         public void Stop()
 138         {
 139             if (_watcher != null)
 140             {
 141                 _watcher.Changed -= OnFileEvent;
 142                 _watcher.Created -= OnFileEvent;
 143                 _watcher.Deleted -= OnFileEvent;
 144                 _watcher.Renamed -= OnRenameEvent;
 145                 _watcher.Dispose();
 146             }
 147             _watcher = null;
 148
 149             if (_agent!=null)
 150                 _agent.Complete();
 151         }
 152
 153         // Enumerate all files in the Pithos directory except those in the Fragment folder
 154         // and files with a .ignore extension
 155         public IEnumerable<string> EnumerateFiles(string searchPattern="*")
 156         {
 157             var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
 158                                  where !Ignore(filePath)
 159                                  select filePath;
 160             return monitoredFiles;
 161         }
 162
 163         public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
 164         {
 165             var rootDir = new DirectoryInfo(RootPath);
 166             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 167                                  where !Ignore(file.FullName)
 168                                  select file;
 169             return monitoredFiles;
 170         }
 171
 172         public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
 173         {
 174             var rootDir = new DirectoryInfo(RootPath);
 175             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 176                                  where !Ignore(file.FullName)
 177                                  select file.AsRelativeUrlTo(RootPath);
 178             return monitoredFiles;
 179         }
 180
 181
 182
 183
 184         private bool Ignore(string filePath)
 185         {
 186             if (filePath.StartsWith(CachePath))
 187                 return true;
 188             if (_ignoreFiles.ContainsKey(filePath.ToLower()))
 189                 return true;
 190             return false;
 191         }
 192
 193         //Post a Change message for all events except rename
 194         void OnFileEvent(object sender, FileSystemEventArgs e)
 195         {
 196             //Ignore events that affect the cache folder
 197             var filePath = e.FullPath;
 198             if (Ignore(filePath))
 199                 return;
 200
 201             _agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
 202         }
 203
 204
 205         //Post a Change message for renames containing the old and new names
 206         void OnRenameEvent(object sender, RenamedEventArgs e)
 207         {
 208             var oldFullPath = e.OldFullPath;
 209             var fullPath = e.FullPath;
 210             if (Ignore(oldFullPath) || Ignore(fullPath))
 211                 return;
 212
 213             _agent.Post(new WorkflowState
 214             {
 215                 AccountInfo=AccountInfo,
 216                 OldPath = oldFullPath,
 217                 OldFileName = e.OldName,
 218                 Path = fullPath,
 219                 FileName = e.Name,
 220                 TriggeringChange = e.ChangeType
 221             });
 222         }
 223
 224
 225
 226         private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
 227         {
 228             {WatcherChangeTypes.Created,FileStatus.Created},
 229             {WatcherChangeTypes.Changed,FileStatus.Modified},
 230             {WatcherChangeTypes.Deleted,FileStatus.Deleted},
 231             {WatcherChangeTypes.Renamed,FileStatus.Renamed}
 232         };
 233
 234         private Dictionary<string,string> _ignoreFiles=new Dictionary<string, string>();
 235
 236         private WorkflowState UpdateFileStatus(WorkflowState state)
 237         {
 238             if (state==null)
 239                 throw new ArgumentNullException("state");
 240             if (String.IsNullOrWhiteSpace(state.Path))
 241                 throw new ArgumentException("The state's Path can't be empty","state");
 242             Contract.EndContractBlock();
 243
 244             var path = state.Path;
 245             var status = _statusDict[state.TriggeringChange];
 246             var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
 247             if (status == oldStatus)
 248             {
 249                 state.Status = status;
 250                 state.Skip = true;
 251                 return state;
 252             }
 253             if (state.Status == FileStatus.Renamed)
 254                 Workflow.ClearFileStatus(path);
 255
 256             state.Status = Workflow.SetFileStatus(path, status);
 257             return state;
 258         }
 259
 260         private WorkflowState UpdateOverlayStatus(WorkflowState state)
 261         {
 262             if (state==null)
 263                 throw new ArgumentNullException("state");
 264             Contract.EndContractBlock();
 265
 266             if (state.Skip)
 267                 return state;
 268
 269             switch (state.Status)
 270             {
 271                 case FileStatus.Created:
 272                 case FileStatus.Modified:
 273                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 274                     break;
 275                 case FileStatus.Deleted:
 276                     //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
 277                     break;
 278                 case FileStatus.Renamed:
 279                     this.StatusKeeper.ClearFileStatus(state.OldPath);
 280                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 281                     break;
 282                 case FileStatus.Unchanged:
 283                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
 284                     break;
 285             }
 286
 287             if (state.Status == FileStatus.Deleted)
 288                 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
 289             else
 290                 NativeMethods.RaiseChangeNotification(state.Path);
 291             return state;
 292         }
 293
 294
 295         private WorkflowState UpdateFileChecksum(WorkflowState state)
 296         {
 297             if (state.Skip)
 298                 return state;
 299
 300             if (state.Status == FileStatus.Deleted)
 301                 return state;
 302
 303             var path = state.Path;
 304             //Skip calculation for folders
 305             if (Directory.Exists(path))
 306                 return state;
 307
 308             var info = new FileInfo(path);
 309             string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
 310             StatusKeeper.UpdateFileChecksum(path, hash);
 311
 312             state.Hash = hash;
 313             return state;
 314         }
 315
 316         //Does the file exist in the container's local folder?
 317         public bool Exists(string relativePath)
 318         {
 319             if (String.IsNullOrWhiteSpace(relativePath))
 320                 throw new ArgumentNullException("relativePath");
 321             //A RootPath must be set before calling this method
 322             if (String.IsNullOrWhiteSpace(RootPath))
 323                 throw new InvalidOperationException("RootPath was not set");
 324             Contract.EndContractBlock();
 325             //Create the absolute path by combining the RootPath with the relativePath
 326             var absolutePath=Path.Combine(RootPath, relativePath);
 327             //Is this a valid file?
 328             if (File.Exists(absolutePath))
 329                 return true;
 330             //Or a directory?
 331             if (Directory.Exists(absolutePath))
 332                 return true;
 333             //Fail if it is neither
 334             return false;
 335         }
 336
 337         public FileInfo GetFileInfo(string relativePath)
 338         {
 339             if (String.IsNullOrWhiteSpace(relativePath))
 340                 throw new ArgumentNullException("relativePath");
 341             //A RootPath must be set before calling this method
 342             if (String.IsNullOrWhiteSpace(RootPath))
 343                 throw new InvalidOperationException("RootPath was not set");
 344             Contract.EndContractBlock();
 345
 346             var absolutePath = Path.Combine(RootPath, relativePath);
 347 //            Debug.Assert(File.Exists(absolutePath),String.Format("Path {0} doesn't exist",absolutePath));
 348
 349             return new FileInfo(absolutePath);
 350
 351         }
 352
 353         public void Delete(string relativePath)
 354         {
 355             var absolutePath = Path.Combine(RootPath, relativePath);
 356             if (File.Exists(absolutePath))
 357             {
 358                 File.Delete(absolutePath);
 359                 _ignoreFiles[absolutePath.ToLower()] = absolutePath.ToLower();
 360             }
 361             StatusKeeper.ClearFileStatus(absolutePath);
 362         }
 363     }
 364 }