code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/FileAgent.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="FileAgent.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System;
  43 using System.Collections.Generic;
  44 using System.ComponentModel.Composition;
  45 using System.Diagnostics;
  46 using System.Diagnostics.Contracts;
  47 using System.IO;
  48 using System.Linq;
  49 using System.Text;
  50 using System.Threading.Tasks;
  51 using Pithos.Interfaces;
  52 using Pithos.Network;
  53 using log4net;
  54 using log4net.Core;
  55
  56 namespace Pithos.Core.Agents
  57 {
  58 //    [Export]
  59     public class FileAgent
  60     {
  61         Agent<WorkflowState> _agent;
  62         private FileSystemWatcher _watcher;
  63         private FileSystemWatcherAdapter _adapter;
  64
  65         //[Import]
  66         public IStatusKeeper StatusKeeper { get; set; }
  67         //[Import]
  68         public IPithosWorkflow Workflow { get; set; }
  69         //[Import]
  70         public WorkflowAgent WorkflowAgent { get; set; }
  71
  72         private AccountInfo AccountInfo { get; set; }
  73
  74         private string RootPath { get;  set; }
  75
  76         private static readonly ILog Log = LogManager.GetLogger("FileAgent");
  77
  78         public void Start(AccountInfo accountInfo,string rootPath)
  79         {
  80             if (accountInfo==null)
  81                 throw new ArgumentNullException("accountInfo");
  82             if (String.IsNullOrWhiteSpace(rootPath))
  83                 throw new ArgumentNullException("rootPath");
  84             if (!Path.IsPathRooted(rootPath))
  85                 throw new ArgumentException("rootPath must be an absolute path","rootPath");
  86             Contract.EndContractBlock();
  87
  88             AccountInfo = accountInfo;
  89             RootPath = rootPath;
  90             _watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true};
  91             _adapter = new FileSystemWatcherAdapter(_watcher);
  92
  93             _adapter.Changed += OnFileEvent;
  94             _adapter.Created += OnFileEvent;
  95             _adapter.Deleted += OnFileEvent;
  96             _adapter.Renamed += OnRenameEvent;
  97             _adapter.Moved += OnMoveEvent;
  98             _watcher.EnableRaisingEvents = true;
  99
 100
 101             _agent = Agent<WorkflowState>.Start(inbox =>
 102             {
 103                 Action loop = null;
 104                 loop = () =>
 105                 {
 106                     var message = inbox.Receive();
 107                     var process=message.Then(Process,inbox.CancellationToken);
 108                     inbox.LoopAsync(process,loop,ex=>
 109                         Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
 110                 };
 111                 loop();
 112             });
 113         }
 114
 115         private Task<object> Process(WorkflowState state)
 116         {
 117             if (state==null)
 118                 throw new ArgumentNullException("state");
 119             Contract.EndContractBlock();
 120
 121             if (Ignore(state.Path))
 122                 return CompletedTask<object>.Default;
 123
 124             var networkState = NetworkGate.GetNetworkState(state.Path);
 125             //Skip if the file is already being downloaded or uploaded and
 126             //the change is create or modify
 127             if (networkState != NetworkOperation.None &&
 128                 (
 129                     state.TriggeringChange == WatcherChangeTypes.Created ||
 130                     state.TriggeringChange == WatcherChangeTypes.Changed
 131                 ))
 132                 return CompletedTask<object>.Default;
 133
 134             try
 135             {
 136                 UpdateFileStatus(state);
 137                 UpdateOverlayStatus(state);
 138                 UpdateFileChecksum(state);
 139                 WorkflowAgent.Post(state);
 140             }
 141             catch (IOException exc)
 142             {
 143                 if (File.Exists(state.Path))
 144                 {
 145                     Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
 146                     _agent.Post(state);
 147                 }
 148                 else
 149                 {
 150                     Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
 151                 }
 152             }
 153             catch (Exception exc)
 154             {
 155                 Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
 156                                state.Path, exc);
 157             }
 158             return CompletedTask<object>.Default;
 159         }
 160
 161         public bool Pause
 162         {
 163             get { return _watcher == null || !_watcher.EnableRaisingEvents; }
 164             set
 165             {
 166                 if (_watcher != null)
 167                     _watcher.EnableRaisingEvents = !value;
 168             }
 169         }
 170
 171         public string CachePath { get; set; }
 172
 173         private List<string> _selectivePaths = new List<string>();
 174         public List<string> SelectivePaths
 175         {
 176             get { return _selectivePaths; }
 177             set { _selectivePaths = value; }
 178         }
 179
 180
 181         public void Post(WorkflowState workflowState)
 182         {
 183             if (workflowState == null)
 184                 throw new ArgumentNullException("workflowState");
 185             Contract.EndContractBlock();
 186
 187             _agent.Post(workflowState);
 188         }
 189
 190         public void Stop()
 191         {
 192             if (_watcher != null)
 193             {
 194                 _watcher.Changed -= OnFileEvent;
 195                 _watcher.Created -= OnFileEvent;
 196                 _watcher.Deleted -= OnFileEvent;
 197                 _watcher.Renamed -= OnRenameEvent;
 198                 _watcher.Dispose();
 199             }
 200             _watcher = null;
 201
 202             if (_agent!=null)
 203                 _agent.Stop();
 204         }
 205
 206         // Enumerate all files in the Pithos directory except those in the Fragment folder
 207         // and files with a .ignore extension
 208         public IEnumerable<string> EnumerateFiles(string searchPattern="*")
 209         {
 210             var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
 211                                  where !Ignore(filePath)
 212                                  select filePath;
 213             return monitoredFiles;
 214         }
 215
 216         public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
 217         {
 218             var rootDir = new DirectoryInfo(RootPath);
 219             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 220                                  where !Ignore(file.FullName)
 221                                  select file;
 222             return monitoredFiles;
 223         }
 224
 225         public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
 226         {
 227             var rootDir = new DirectoryInfo(RootPath);
 228             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 229                                  where !Ignore(file.FullName)
 230                                  select file.AsRelativeUrlTo(RootPath);
 231             return monitoredFiles;
 232         }
 233
 234         public IEnumerable<string> EnumerateFilesSystemInfosAsRelativeUrls(string searchPattern="*")
 235         {
 236             var rootDir = new DirectoryInfo(RootPath);
 237             var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories)
 238                                  where !Ignore(file.FullName)
 239                                  select file.AsRelativeUrlTo(RootPath);
 240             return monitoredFiles;
 241         }
 242
 243
 244
 245
 246         private bool Ignore(string filePath)
 247         {
 248             //Ignore all first-level directories and files (ie at the container folders level)
 249             if (FoundBelowRoot(filePath, RootPath,1))
 250                 return true;
 251
 252             //Ignore first-level items under the "others" folder (ie at the accounts folders level).
 253             var othersPath = Path.Combine(RootPath, FolderConstants.OthersFolder);
 254             if (FoundBelowRoot(filePath, othersPath,1))
 255                 return true;
 256
 257             //Ignore second-level (container) folders under the "others" folder (ie at the container folders level).
 258             if (FoundBelowRoot(filePath, othersPath,2))
 259                 return true;
 260
 261
 262             //Ignore anything happening in the cache path
 263             if (filePath.StartsWith(CachePath))
 264                 return true;
 265             if (_ignoreFiles.ContainsKey(filePath.ToLower()))
 266                 return true;
 267
 268             //If selective synchronization is defined,
 269             if (SelectivePaths.Count>0)
 270             {
 271                 //Abort if the file is not located under any of the selected folders
 272                 if (!SelectivePaths.Any(filePath.StartsWith))
 273                     return true;
 274             }
 275
 276             return false;
 277         }
 278
 279 /*        private static bool FoundInRoot(string filePath, string rootPath)
 280         {
 281             //var rootDirectory = new DirectoryInfo(rootPath);
 282
 283             //If the paths are equal, return true
 284             if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
 285                 return true;
 286
 287             //If the filepath is below the root path
 288             if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
 289             {
 290                 //Get the relative path
 291                 var relativePath = filePath.Substring(rootPath.Length + 1);
 292                 //If the relativePath does NOT contains a path separator, we found a match
 293                 return (!relativePath.Contains(@"\"));
 294             }
 295
 296             //If the filepath is not under the root path, return false
 297             return false;
 298         }*/
 299
 300
 301         private static bool FoundBelowRoot(string filePath, string rootPath,int level)
 302         {
 303             //var rootDirectory = new DirectoryInfo(rootPath);
 304
 305             //If the paths are equal, return true
 306             if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
 307                 return true;
 308
 309             //If the filepath is below the root path
 310             if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
 311             {
 312                 //Get the relative path
 313                 var relativePath = filePath.Substring(rootPath.Length + 1);
 314                 //If the relativePath does NOT contains a path separator, we found a match
 315                 var levels=relativePath.ToCharArray().Count(c=>c=='\\')+1;
 316                 return levels==level;
 317             }
 318
 319             //If the filepath is not under the root path, return false
 320             return false;
 321         }
 322
 323         //Post a Change message for all events except rename
 324         void OnFileEvent(object sender, FileSystemEventArgs e)
 325         {
 326             //Ignore events that affect the cache folder
 327             var filePath = e.FullPath;
 328             if (Ignore(filePath))
 329                 return;
 330
 331             _agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
 332         }
 333
 334
 335         //Post a Change message for renames containing the old and new names
 336         void OnRenameEvent(object sender, RenamedEventArgs e)
 337         {
 338             var oldFullPath = e.OldFullPath;
 339             var fullPath = e.FullPath;
 340             if (Ignore(oldFullPath) || Ignore(fullPath))
 341                 return;
 342
 343             _agent.Post(new WorkflowState
 344             {
 345                 AccountInfo=AccountInfo,
 346                 OldPath = oldFullPath,
 347                 OldFileName = e.OldName,
 348                 Path = fullPath,
 349                 FileName = e.Name,
 350                 TriggeringChange = e.ChangeType
 351             });
 352         }
 353
 354         //Post a Change message for renames containing the old and new names
 355         void OnMoveEvent(object sender, MovedEventArgs e)
 356         {
 357             var oldFullPath = e.OldFullPath;
 358             var fullPath = e.FullPath;
 359             if (Ignore(oldFullPath) || Ignore(fullPath))
 360                 return;
 361
 362             _agent.Post(new WorkflowState
 363             {
 364                 AccountInfo=AccountInfo,
 365                 OldPath = oldFullPath,
 366                 OldFileName = e.OldName,
 367                 Path = fullPath,
 368                 FileName = e.Name,
 369                 TriggeringChange = e.ChangeType
 370             });
 371         }
 372
 373
 374
 375         private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
 376                                                                              {
 377             {WatcherChangeTypes.Created,FileStatus.Created},
 378             {WatcherChangeTypes.Changed,FileStatus.Modified},
 379             {WatcherChangeTypes.Deleted,FileStatus.Deleted},
 380             {WatcherChangeTypes.Renamed,FileStatus.Renamed}
 381         };
 382
 383         private Dictionary<string, string> _ignoreFiles=new Dictionary<string, string>();
 384
 385         private WorkflowState UpdateFileStatus(WorkflowState state)
 386         {
 387             if (state==null)
 388                 throw new ArgumentNullException("state");
 389             if (String.IsNullOrWhiteSpace(state.Path))
 390                 throw new ArgumentException("The state's Path can't be empty","state");
 391             Contract.EndContractBlock();
 392
 393             var path = state.Path;
 394             var status = _statusDict[state.TriggeringChange];
 395             var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
 396             if (status == oldStatus)
 397             {
 398                 state.Status = status;
 399                 state.Skip = true;
 400                 return state;
 401             }
 402             if (state.Status == FileStatus.Renamed)
 403                 Workflow.ClearFileStatus(path);
 404
 405             state.Status = Workflow.SetFileStatus(path, status);
 406             return state;
 407         }
 408
 409         private WorkflowState UpdateOverlayStatus(WorkflowState state)
 410         {
 411             if (state==null)
 412                 throw new ArgumentNullException("state");
 413             Contract.EndContractBlock();
 414
 415             if (state.Skip)
 416                 return state;
 417
 418             switch (state.Status)
 419             {
 420                 case FileStatus.Created:
 421                 case FileStatus.Modified:
 422                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 423                     break;
 424                 case FileStatus.Deleted:
 425                     //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
 426                     break;
 427                 case FileStatus.Renamed:
 428                     this.StatusKeeper.ClearFileStatus(state.OldPath);
 429                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 430                     break;
 431                 case FileStatus.Unchanged:
 432                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
 433                     break;
 434             }
 435
 436             if (state.Status == FileStatus.Deleted)
 437                 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
 438             else
 439                 NativeMethods.RaiseChangeNotification(state.Path);
 440             return state;
 441         }
 442
 443
 444         private WorkflowState UpdateFileChecksum(WorkflowState state)
 445         {
 446             if (state.Skip)
 447                 return state;
 448
 449             if (state.Status == FileStatus.Deleted)
 450                 return state;
 451
 452             var path = state.Path;
 453             //Skip calculation for folders
 454             if (Directory.Exists(path))
 455                 return state;
 456
 457             var info = new FileInfo(path);
 458             string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
 459             StatusKeeper.UpdateFileChecksum(path, hash);
 460
 461             state.Hash = hash;
 462             return state;
 463         }
 464
 465         //Does the file exist in the container's local folder?
 466         public bool Exists(string relativePath)
 467         {
 468             if (String.IsNullOrWhiteSpace(relativePath))
 469                 throw new ArgumentNullException("relativePath");
 470             //A RootPath must be set before calling this method
 471             if (String.IsNullOrWhiteSpace(RootPath))
 472                 throw new InvalidOperationException("RootPath was not set");
 473             Contract.EndContractBlock();
 474             //Create the absolute path by combining the RootPath with the relativePath
 475             var absolutePath=Path.Combine(RootPath, relativePath);
 476             //Is this a valid file?
 477             if (File.Exists(absolutePath))
 478                 return true;
 479             //Or a directory?
 480             if (Directory.Exists(absolutePath))
 481                 return true;
 482             //Fail if it is neither
 483             return false;
 484         }
 485
 486         public static FileAgent GetFileAgent(AccountInfo accountInfo)
 487         {
 488             return GetFileAgent(accountInfo.AccountPath);
 489         }
 490
 491         public static FileAgent GetFileAgent(string rootPath)
 492         {
 493             return AgentLocator<FileAgent>.Get(rootPath.ToLower());
 494         }
 495
 496
 497         public FileSystemInfo GetFileSystemInfo(string relativePath)
 498         {
 499             if (String.IsNullOrWhiteSpace(relativePath))
 500                 throw new ArgumentNullException("relativePath");
 501             //A RootPath must be set before calling this method
 502             if (String.IsNullOrWhiteSpace(RootPath))
 503                 throw new InvalidOperationException("RootPath was not set");
 504             Contract.EndContractBlock();
 505
 506             var absolutePath = Path.Combine(RootPath, relativePath);
 507
 508             if (Directory.Exists(absolutePath))
 509                 return new DirectoryInfo(absolutePath).WithProperCapitalization();
 510             else
 511                 return new FileInfo(absolutePath).WithProperCapitalization();
 512
 513         }
 514
 515         public void Delete(string relativePath)
 516         {
 517             var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
 518             if (File.Exists(absolutePath))
 519             {
 520                 try
 521                 {
 522                     File.Delete(absolutePath);
 523                 }
 524                 //The file may have been deleted by another thread. Just ignore the relevant exception
 525                 catch (FileNotFoundException) { }
 526             }
 527             else if (Directory.Exists(absolutePath))
 528             {
 529                 try
 530                 {
 531                     Directory.Delete(absolutePath, true);
 532                 }
 533                 //The directory may have been deleted by another thread. Just ignore the relevant exception
 534                 catch (DirectoryNotFoundException){}
 535             }
 536
 537             //_ignoreFiles[absolutePath] = absolutePath;
 538             StatusKeeper.ClearFileStatus(absolutePath);
 539         }
 540     }
 541 }