code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/FileAgent.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="FileAgent.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System;
  43 using System.Collections.Generic;
  44 using System.ComponentModel.Composition;
  45 using System.Diagnostics;
  46 using System.Diagnostics.Contracts;
  47 using System.IO;
  48 using System.Linq;
  49 using System.Text;
  50 using System.Threading.Tasks;
  51 using Pithos.Interfaces;
  52 using Pithos.Network;
  53 using log4net;
  54 using log4net.Core;
  55
  56 namespace Pithos.Core.Agents
  57 {
  58 //    [Export]
  59     public class FileAgent
  60     {
  61         Agent<WorkflowState> _agent;
  62         private FileSystemWatcher _watcher;
  63         private FileSystemWatcherAdapter _adapter;
  64
  65         //[Import]
  66         public IStatusKeeper StatusKeeper { get; set; }
  67         //[Import]
  68         public IPithosWorkflow Workflow { get; set; }
  69         //[Import]
  70         public WorkflowAgent WorkflowAgent { get; set; }
  71
  72         private AccountInfo AccountInfo { get; set; }
  73
  74         private string RootPath { get;  set; }
  75
  76         private static readonly ILog Log = LogManager.GetLogger("FileAgent");
  77
  78         public void Start(AccountInfo accountInfo,string rootPath)
  79         {
  80             if (accountInfo==null)
  81                 throw new ArgumentNullException("accountInfo");
  82             if (String.IsNullOrWhiteSpace(rootPath))
  83                 throw new ArgumentNullException("rootPath");
  84             if (!Path.IsPathRooted(rootPath))
  85                 throw new ArgumentException("rootPath must be an absolute path","rootPath");
  86             Contract.EndContractBlock();
  87
  88             AccountInfo = accountInfo;
  89             RootPath = rootPath;
  90             _watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true};
  91             _adapter = new FileSystemWatcherAdapter(_watcher);
  92
  93             _adapter.Changed += OnFileEvent;
  94             _adapter.Created += OnFileEvent;
  95             _adapter.Deleted += OnFileEvent;
  96             _adapter.Renamed += OnRenameEvent;
  97             _adapter.Moved += OnMoveEvent;
  98             _watcher.EnableRaisingEvents = true;
  99
 100
 101             _agent = Agent<WorkflowState>.Start(inbox =>
 102             {
 103                 Action loop = null;
 104                 loop = () =>
 105                 {
 106                     var message = inbox.Receive();
 107                     var process=message.Then(Process,inbox.CancellationToken);
 108                     inbox.LoopAsync(process,loop,ex=>
 109                         Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
 110                 };
 111                 loop();
 112             });
 113         }
 114
 115         private Task<object> Process(WorkflowState state)
 116         {
 117             if (state==null)
 118                 throw new ArgumentNullException("state");
 119             Contract.EndContractBlock();
 120
 121             if (Ignore(state.Path))
 122                 return CompletedTask<object>.Default;
 123
 124             var networkState = NetworkGate.GetNetworkState(state.Path);
 125             //Skip if the file is already being downloaded or uploaded and
 126             //the change is create or modify
 127             if (networkState != NetworkOperation.None &&
 128                 (
 129                     state.TriggeringChange == WatcherChangeTypes.Created ||
 130                     state.TriggeringChange == WatcherChangeTypes.Changed
 131                 ))
 132                 return CompletedTask<object>.Default;
 133
 134             try
 135             {
 136                 UpdateFileStatus(state);
 137                 UpdateOverlayStatus(state);
 138                 UpdateFileChecksum(state);
 139                 WorkflowAgent.Post(state);
 140             }
 141             catch (IOException exc)
 142             {
 143                 if (File.Exists(state.Path))
 144                 {
 145                     Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
 146                     _agent.Post(state);
 147                 }
 148                 else
 149                 {
 150                     Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
 151                 }
 152             }
 153             catch (Exception exc)
 154             {
 155                 Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
 156                                state.Path, exc);
 157             }
 158             return CompletedTask<object>.Default;
 159         }
 160
 161         public bool Pause
 162         {
 163             get { return _watcher == null || !_watcher.EnableRaisingEvents; }
 164             set
 165             {
 166                 if (_watcher != null)
 167                     _watcher.EnableRaisingEvents = !value;
 168             }
 169         }
 170
 171         public string CachePath { get; set; }
 172
 173         private List<string> _selectivePaths = new List<string>();
 174         public List<string> SelectivePaths
 175         {
 176             get { return _selectivePaths; }
 177             set { _selectivePaths = value; }
 178         }
 179
 180
 181         public void Post(WorkflowState workflowState)
 182         {
 183             if (workflowState == null)
 184                 throw new ArgumentNullException("workflowState");
 185             Contract.EndContractBlock();
 186
 187             _agent.Post(workflowState);
 188         }
 189
 190         public void Stop()
 191         {
 192             if (_watcher != null)
 193             {
 194                 _watcher.Changed -= OnFileEvent;
 195                 _watcher.Created -= OnFileEvent;
 196                 _watcher.Deleted -= OnFileEvent;
 197                 _watcher.Renamed -= OnRenameEvent;
 198                 _watcher.Dispose();
 199             }
 200             _watcher = null;
 201
 202             if (_agent!=null)
 203                 _agent.Stop();
 204         }
 205
 206         // Enumerate all files in the Pithos directory except those in the Fragment folder
 207         // and files with a .ignore extension
 208         public IEnumerable<string> EnumerateFiles(string searchPattern="*")
 209         {
 210             var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
 211                                  where !Ignore(filePath)
 212                                  select filePath;
 213             return monitoredFiles;
 214         }
 215
 216         public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
 217         {
 218             var rootDir = new DirectoryInfo(RootPath);
 219             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 220                                  where !Ignore(file.FullName)
 221                                  select file;
 222             return monitoredFiles;
 223         }
 224
 225         public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
 226         {
 227             var rootDir = new DirectoryInfo(RootPath);
 228             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 229                                  where !Ignore(file.FullName)
 230                                  select file.AsRelativeUrlTo(RootPath);
 231             return monitoredFiles;
 232         }
 233
 234         public IEnumerable<string> EnumerateFilesSystemInfosAsRelativeUrls(string searchPattern="*")
 235         {
 236             var rootDir = new DirectoryInfo(RootPath);
 237             var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories)
 238                                  where !Ignore(file.FullName)
 239                                  select file.AsRelativeUrlTo(RootPath);
 240             return monitoredFiles;
 241         }
 242
 243
 244
 245
 246         private bool Ignore(string filePath)
 247         {
 248             //Ignore all first-level directories and files (ie at the container folders level)
 249             if (FoundBelowRoot(filePath, RootPath,1))
 250                 return true;
 251
 252             //Ignore first-level items under the "others" folder (ie at the accounts folders level).
 253             var othersPath = Path.Combine(RootPath, FolderConstants.OthersFolder);
 254             if (FoundBelowRoot(filePath, othersPath,1))
 255                 return true;
 256
 257             //Ignore second-level (container) folders under the "others" folder (ie at the container folders level).
 258             if (FoundBelowRoot(filePath, othersPath,2))
 259                 return true;
 260
 261
 262             //Ignore anything happening in the cache path
 263             if (filePath.StartsWith(CachePath))
 264                 return true;
 265             if (_ignoreFiles.ContainsKey(filePath.ToLower()))
 266                 return true;
 267
 268             //Ignore if selective synchronization is defined,
 269             return SelectivePaths.Count > 0
 270                 //And the target file is not below any of the selective paths
 271                 && !SelectivePaths.Any(filePath.IsAtOrDirectlyBelow);
 272         }
 273
 274 /*        private static bool FoundInRoot(string filePath, string rootPath)
 275         {
 276             //var rootDirectory = new DirectoryInfo(rootPath);
 277
 278             //If the paths are equal, return true
 279             if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
 280                 return true;
 281
 282             //If the filepath is below the root path
 283             if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
 284             {
 285                 //Get the relative path
 286                 var relativePath = filePath.Substring(rootPath.Length + 1);
 287                 //If the relativePath does NOT contains a path separator, we found a match
 288                 return (!relativePath.Contains(@"\"));
 289             }
 290
 291             //If the filepath is not under the root path, return false
 292             return false;
 293         }*/
 294
 295
 296         private static bool FoundBelowRoot(string filePath, string rootPath,int level)
 297         {
 298             //var rootDirectory = new DirectoryInfo(rootPath);
 299
 300             //If the paths are equal, return true
 301             if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
 302                 return true;
 303
 304             //If the filepath is below the root path
 305             if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
 306             {
 307                 //Get the relative path
 308                 var relativePath = filePath.Substring(rootPath.Length + 1);
 309                 //If the relativePath does NOT contains a path separator, we found a match
 310                 var levels=relativePath.ToCharArray().Count(c=>c=='\\')+1;
 311                 return levels==level;
 312             }
 313
 314             //If the filepath is not under the root path, return false
 315             return false;
 316         }
 317
 318         //Post a Change message for all events except rename
 319         void OnFileEvent(object sender, FileSystemEventArgs e)
 320         {
 321             //Ignore events that affect the cache folder
 322             var filePath = e.FullPath;
 323             if (Ignore(filePath))
 324                 return;
 325
 326             _agent.Post(new WorkflowState{AccountInfo=AccountInfo, Path = filePath, FileName = e.Name, TriggeringChange = e.ChangeType });
 327         }
 328
 329
 330         //Post a Change message for renames containing the old and new names
 331         void OnRenameEvent(object sender, RenamedEventArgs e)
 332         {
 333             var oldFullPath = e.OldFullPath;
 334             var fullPath = e.FullPath;
 335             if (Ignore(oldFullPath) || Ignore(fullPath))
 336                 return;
 337
 338             _agent.Post(new WorkflowState
 339             {
 340                 AccountInfo=AccountInfo,
 341                 OldPath = oldFullPath,
 342                 OldFileName = e.OldName,
 343                 Path = fullPath,
 344                 FileName = e.Name,
 345                 TriggeringChange = e.ChangeType
 346             });
 347         }
 348
 349         //Post a Change message for renames containing the old and new names
 350         void OnMoveEvent(object sender, MovedEventArgs e)
 351         {
 352             var oldFullPath = e.OldFullPath;
 353             var fullPath = e.FullPath;
 354             if (Ignore(oldFullPath) || Ignore(fullPath))
 355                 return;
 356
 357             _agent.Post(new WorkflowState
 358             {
 359                 AccountInfo=AccountInfo,
 360                 OldPath = oldFullPath,
 361                 OldFileName = e.OldName,
 362                 Path = fullPath,
 363                 FileName = e.Name,
 364                 TriggeringChange = e.ChangeType
 365             });
 366         }
 367
 368
 369
 370         private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
 371                                                                              {
 372             {WatcherChangeTypes.Created,FileStatus.Created},
 373             {WatcherChangeTypes.Changed,FileStatus.Modified},
 374             {WatcherChangeTypes.Deleted,FileStatus.Deleted},
 375             {WatcherChangeTypes.Renamed,FileStatus.Renamed}
 376         };
 377
 378         private Dictionary<string, string> _ignoreFiles=new Dictionary<string, string>();
 379
 380         private WorkflowState UpdateFileStatus(WorkflowState state)
 381         {
 382             if (state==null)
 383                 throw new ArgumentNullException("state");
 384             if (String.IsNullOrWhiteSpace(state.Path))
 385                 throw new ArgumentException("The state's Path can't be empty","state");
 386             Contract.EndContractBlock();
 387
 388             var path = state.Path;
 389             var status = _statusDict[state.TriggeringChange];
 390             var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
 391             if (status == oldStatus)
 392             {
 393                 state.Status = status;
 394                 state.Skip = true;
 395                 return state;
 396             }
 397             if (state.Status == FileStatus.Renamed)
 398                 Workflow.ClearFileStatus(path);
 399
 400             state.Status = Workflow.SetFileStatus(path, status);
 401             return state;
 402         }
 403
 404         private WorkflowState UpdateOverlayStatus(WorkflowState state)
 405         {
 406             if (state==null)
 407                 throw new ArgumentNullException("state");
 408             Contract.EndContractBlock();
 409
 410             if (state.Skip)
 411                 return state;
 412
 413             switch (state.Status)
 414             {
 415                 case FileStatus.Created:
 416                 case FileStatus.Modified:
 417                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 418                     break;
 419                 case FileStatus.Deleted:
 420                     //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
 421                     break;
 422                 case FileStatus.Renamed:
 423                     this.StatusKeeper.ClearFileStatus(state.OldPath);
 424                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
 425                     break;
 426                 case FileStatus.Unchanged:
 427                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
 428                     break;
 429             }
 430
 431             if (state.Status == FileStatus.Deleted)
 432                 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
 433             else
 434                 NativeMethods.RaiseChangeNotification(state.Path);
 435             return state;
 436         }
 437
 438
 439         private WorkflowState UpdateFileChecksum(WorkflowState state)
 440         {
 441             if (state.Skip)
 442                 return state;
 443
 444             if (state.Status == FileStatus.Deleted)
 445                 return state;
 446
 447             var path = state.Path;
 448             //Skip calculation for folders
 449             if (Directory.Exists(path))
 450                 return state;
 451
 452             var info = new FileInfo(path);
 453             string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
 454             StatusKeeper.UpdateFileChecksum(path, hash);
 455
 456             state.Hash = hash;
 457             return state;
 458         }
 459
 460         //Does the file exist in the container's local folder?
 461         public bool Exists(string relativePath)
 462         {
 463             if (String.IsNullOrWhiteSpace(relativePath))
 464                 throw new ArgumentNullException("relativePath");
 465             //A RootPath must be set before calling this method
 466             if (String.IsNullOrWhiteSpace(RootPath))
 467                 throw new InvalidOperationException("RootPath was not set");
 468             Contract.EndContractBlock();
 469             //Create the absolute path by combining the RootPath with the relativePath
 470             var absolutePath=Path.Combine(RootPath, relativePath);
 471             //Is this a valid file?
 472             if (File.Exists(absolutePath))
 473                 return true;
 474             //Or a directory?
 475             if (Directory.Exists(absolutePath))
 476                 return true;
 477             //Fail if it is neither
 478             return false;
 479         }
 480
 481         public static FileAgent GetFileAgent(AccountInfo accountInfo)
 482         {
 483             return GetFileAgent(accountInfo.AccountPath);
 484         }
 485
 486         public static FileAgent GetFileAgent(string rootPath)
 487         {
 488             return AgentLocator<FileAgent>.Get(rootPath.ToLower());
 489         }
 490
 491
 492         public FileSystemInfo GetFileSystemInfo(string relativePath)
 493         {
 494             if (String.IsNullOrWhiteSpace(relativePath))
 495                 throw new ArgumentNullException("relativePath");
 496             //A RootPath must be set before calling this method
 497             if (String.IsNullOrWhiteSpace(RootPath))
 498                 throw new InvalidOperationException("RootPath was not set");
 499             Contract.EndContractBlock();
 500
 501             var absolutePath = Path.Combine(RootPath, relativePath);
 502
 503             if (Directory.Exists(absolutePath))
 504                 return new DirectoryInfo(absolutePath).WithProperCapitalization();
 505             else
 506                 return new FileInfo(absolutePath).WithProperCapitalization();
 507
 508         }
 509
 510         public void Delete(string relativePath)
 511         {
 512             var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
 513             if (File.Exists(absolutePath))
 514             {
 515                 try
 516                 {
 517                     File.Delete(absolutePath);
 518                 }
 519                 //The file may have been deleted by another thread. Just ignore the relevant exception
 520                 catch (FileNotFoundException) { }
 521             }
 522             else if (Directory.Exists(absolutePath))
 523             {
 524                 try
 525                 {
 526                     Directory.Delete(absolutePath, true);
 527                 }
 528                 //The directory may have been deleted by another thread. Just ignore the relevant exception
 529                 catch (DirectoryNotFoundException){}
 530             }
 531
 532             //_ignoreFiles[absolutePath] = absolutePath;
 533             StatusKeeper.ClearFileStatus(absolutePath);
 534         }
 535     }
 536 }