code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/FileAgent.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="FileAgent.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System;
  43 using System.Collections.Generic;
  44 using System.Diagnostics.Contracts;
  45 using System.IO;
  46 using System.Linq;
  47 using System.Reflection;
  48 using System.Threading.Tasks;
  49 using Pithos.Interfaces;
  50 using Pithos.Network;
  51 using log4net;
  52
  53 namespace Pithos.Core.Agents
  54 {
  55 //    [Export]
  56     public class FileAgent
  57     {
  58         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  59
  60         Agent<WorkflowState> _agent;
  61         private FileSystemWatcher _watcher;
  62         private FileSystemWatcherAdapter _adapter;
  63
  64         //[Import]
  65         public IStatusKeeper StatusKeeper { get; set; }
  66
  67         public IStatusNotification StatusNotification { get; set; }
  68         //[Import]
  69         public IPithosWorkflow Workflow { get; set; }
  70         //[Import]
  71         public WorkflowAgent WorkflowAgent { get; set; }
  72
  73         private AccountInfo AccountInfo { get; set; }
  74
  75         internal string RootPath { get;  set; }
  76
  77         private FileEventIdleBatch _eventIdleBatch;
  78
  79         public TimeSpan IdleTimeout { get; set; }
  80
  81
  82         private void ProcessBatchedEvents(Dictionary<string, FileSystemEventArgs[]> fileEvents)
  83         {
  84             StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Uploading {0} files",fileEvents.Count));
  85             foreach (var fileEvent in fileEvents)
  86             {
  87                 var filePath = fileEvent.Key;
  88                 var changes = fileEvent.Value;
  89
  90                 if (Ignore(filePath)) continue;
  91
  92                 foreach (var change in changes)
  93                 {
  94                     if (change.ChangeType == WatcherChangeTypes.Renamed)
  95                     {
  96                         var rename = (MovedEventArgs) change;
  97                         _agent.Post(new WorkflowState
  98                                         {
  99                                             AccountInfo = AccountInfo,
 100                                             OldPath = rename.OldFullPath,
 101                                             OldFileName = Path.GetFileName(rename.OldName),
 102                                             Path = rename.FullPath,
 103                                             FileName = Path.GetFileName(rename.Name),
 104                                             TriggeringChange = rename.ChangeType
 105                                         });
 106                     }
 107                     else
 108                         _agent.Post(new WorkflowState
 109                         {
 110                             AccountInfo = AccountInfo,
 111                             Path = change.FullPath,
 112                             FileName = Path.GetFileName(change.Name),
 113                             TriggeringChange = change.ChangeType
 114                         });
 115                 }
 116             }
 117             StatusNotification.SetPithosStatus(PithosStatus.LocalComplete);
 118         }
 119
 120         public void Start(AccountInfo accountInfo,string rootPath)
 121         {
 122             if (accountInfo==null)
 123                 throw new ArgumentNullException("accountInfo");
 124             if (String.IsNullOrWhiteSpace(rootPath))
 125                 throw new ArgumentNullException("rootPath");
 126             if (!Path.IsPathRooted(rootPath))
 127                 throw new ArgumentException("rootPath must be an absolute path","rootPath");
 128             if (IdleTimeout == null)
 129                 throw new InvalidOperationException("IdleTimeout must have a valid value");
 130             Contract.EndContractBlock();
 131
 132             AccountInfo = accountInfo;
 133             RootPath = rootPath;
 134
 135             _eventIdleBatch = new FileEventIdleBatch((int)IdleTimeout.TotalMilliseconds, ProcessBatchedEvents);
 136
 137             _watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true,InternalBufferSize=8*4096};
 138             _adapter = new FileSystemWatcherAdapter(_watcher);
 139
 140             _adapter.Changed += OnFileEvent;
 141             _adapter.Created += OnFileEvent;
 142             _adapter.Deleted += OnFileEvent;
 143             //_adapter.Renamed += OnRenameEvent;
 144             _adapter.Moved += OnMoveEvent;
 145             _watcher.EnableRaisingEvents = true;
 146
 147
 148             _agent = Agent<WorkflowState>.Start(inbox =>
 149             {
 150                 Action loop = null;
 151                 loop = () =>
 152                 {
 153                     var message = inbox.Receive();
 154                     var process=message.Then(Process,inbox.CancellationToken);
 155                     inbox.LoopAsync(process,loop,ex=>
 156                         Log.ErrorFormat("[ERROR] File Event Processing:\r{0}", ex));
 157                 };
 158                 loop();
 159             });
 160         }
 161
 162         private Task<object> Process(WorkflowState state)
 163         {
 164             if (state==null)
 165                 throw new ArgumentNullException("state");
 166             Contract.EndContractBlock();
 167
 168             if (Ignore(state.Path))
 169                 return CompletedTask<object>.Default;
 170
 171             var networkState = NetworkGate.GetNetworkState(state.Path);
 172             //Skip if the file is already being downloaded or uploaded and
 173             //the change is create or modify
 174             if (networkState != NetworkOperation.None &&
 175                 (
 176                     state.TriggeringChange == WatcherChangeTypes.Created ||
 177                     state.TriggeringChange == WatcherChangeTypes.Changed
 178                 ))
 179                 return CompletedTask<object>.Default;
 180
 181             try
 182             {
 183                 //StatusKeeper.EnsureFileState(state.Path);
 184
 185                 UpdateFileStatus(state);
 186                 UpdateOverlayStatus(state);
 187                 UpdateFileChecksum(state);
 188                 WorkflowAgent.Post(state);
 189             }
 190             catch (IOException exc)
 191             {
 192                 if (File.Exists(state.Path))
 193                 {
 194                     Log.WarnFormat("File access error occured, retrying {0}\n{1}", state.Path, exc);
 195                     _agent.Post(state);
 196                 }
 197                 else
 198                 {
 199                     Log.WarnFormat("File {0} does not exist. Will be ignored\n{1}", state.Path, exc);
 200                 }
 201             }
 202             catch (Exception exc)
 203             {
 204                 Log.WarnFormat("Error occured while indexing{0}. The file will be skipped\n{1}",
 205                                state.Path, exc);
 206             }
 207             return CompletedTask<object>.Default;
 208         }
 209
 210         public bool Pause
 211         {
 212             get { return _watcher == null || !_watcher.EnableRaisingEvents; }
 213             set
 214             {
 215                 if (_watcher != null)
 216                     _watcher.EnableRaisingEvents = !value;
 217             }
 218         }
 219
 220         public string CachePath { get; set; }
 221
 222         /*private List<string> _selectivePaths = new List<string>();
 223         public List<string> SelectivePaths
 224         {
 225             get { return _selectivePaths; }
 226             set { _selectivePaths = value; }
 227         }
 228 */
 229         public Selectives Selectives { get; set; }
 230
 231
 232         public void Post(WorkflowState workflowState)
 233         {
 234             if (workflowState == null)
 235                 throw new ArgumentNullException("workflowState");
 236             Contract.EndContractBlock();
 237
 238             _agent.Post(workflowState);
 239         }
 240
 241         public void Stop()
 242         {
 243             if (_watcher != null)
 244             {
 245                 _watcher.Dispose();
 246             }
 247             _watcher = null;
 248
 249             if (_agent!=null)
 250                 _agent.Stop();
 251         }
 252
 253         // Enumerate all files in the Pithos directory except those in the Fragment folder
 254         // and files with a .ignore extension
 255         public IEnumerable<string> EnumerateFiles(string searchPattern="*")
 256         {
 257             var monitoredFiles = from filePath in Directory.EnumerateFileSystemEntries(RootPath, searchPattern, SearchOption.AllDirectories)
 258                                  where !Ignore(filePath)
 259                                  select filePath;
 260             return monitoredFiles;
 261         }
 262
 263         public IEnumerable<FileInfo> EnumerateFileInfos(string searchPattern="*")
 264         {
 265             var rootDir = new DirectoryInfo(RootPath);
 266             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 267                                  where !Ignore(file.FullName)
 268                                  select file;
 269             return monitoredFiles;
 270         }
 271
 272         public IEnumerable<string> EnumerateFilesAsRelativeUrls(string searchPattern="*")
 273         {
 274             var rootDir = new DirectoryInfo(RootPath);
 275             var monitoredFiles = from file in rootDir.EnumerateFiles(searchPattern, SearchOption.AllDirectories)
 276                                  where !Ignore(file.FullName)
 277                                  select file.AsRelativeUrlTo(RootPath);
 278             return monitoredFiles;
 279         }
 280
 281         public IEnumerable<string> EnumerateFilesSystemInfosAsRelativeUrls(string searchPattern="*")
 282         {
 283             var rootDir = new DirectoryInfo(RootPath);
 284             var monitoredFiles = from file in rootDir.EnumerateFileSystemInfos(searchPattern, SearchOption.AllDirectories)
 285                                  where !Ignore(file.FullName)
 286                                  select file.AsRelativeUrlTo(RootPath);
 287             return monitoredFiles;
 288         }
 289
 290
 291
 292
 293         private bool Ignore(string filePath)
 294         {
 295             //Ignore all first-level directories and files (ie at the container folders level)
 296             if (FoundBelowRoot(filePath, RootPath,1))
 297                 return true;
 298
 299             //Ignore first-level items under the "others" folder (ie at the accounts folders level).
 300             var othersPath = Path.Combine(RootPath, FolderConstants.OthersFolder);
 301             if (FoundBelowRoot(filePath, othersPath,1))
 302                 return true;
 303
 304             //Ignore second-level (container) folders under the "others" folder (ie at the container folders level).
 305             if (FoundBelowRoot(filePath, othersPath,2))
 306                 return true;
 307
 308
 309             //Ignore anything happening in the cache path
 310             if (filePath.StartsWith(CachePath))
 311                 return true;
 312             if (_ignoreFiles.ContainsKey(filePath.ToLower()))
 313                 return true;
 314
 315             //Ignore if selective synchronization is defined,
 316                 //And the target file is not below any of the selective paths
 317             return !Selectives.IsSelected(AccountInfo, filePath);
 318         }
 319
 320 /*        private static bool FoundInRoot(string filePath, string rootPath)
 321         {
 322             //var rootDirectory = new DirectoryInfo(rootPath);
 323
 324             //If the paths are equal, return true
 325             if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
 326                 return true;
 327
 328             //If the filepath is below the root path
 329             if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
 330             {
 331                 //Get the relative path
 332                 var relativePath = filePath.Substring(rootPath.Length + 1);
 333                 //If the relativePath does NOT contains a path separator, we found a match
 334                 return (!relativePath.Contains(@"\"));
 335             }
 336
 337             //If the filepath is not under the root path, return false
 338             return false;
 339         }*/
 340
 341
 342         private static bool FoundBelowRoot(string filePath, string rootPath,int level)
 343         {
 344             //var rootDirectory = new DirectoryInfo(rootPath);
 345
 346             //If the paths are equal, return true
 347             if (filePath.Equals(rootPath, StringComparison.InvariantCultureIgnoreCase))
 348                 return true;
 349
 350             //If the filepath is below the root path
 351             if (filePath.StartsWith(rootPath,StringComparison.InvariantCulture))
 352             {
 353                 //Get the relative path
 354                 var relativePath = filePath.Substring(rootPath.Length + 1);
 355                 //If the relativePath does NOT contains a path separator, we found a match
 356                 var levels=relativePath.ToCharArray().Count(c=>c=='\\')+1;
 357                 return levels==level;
 358             }
 359
 360             //If the filepath is not under the root path, return false
 361             return false;
 362         }
 363
 364         //Post a Change message for all events except rename
 365         void OnFileEvent(object sender, FileSystemEventArgs e)
 366         {
 367             //Ignore events that affect the cache folder
 368             var filePath = e.FullPath;
 369             if (Ignore(filePath))
 370                 return;
 371             _eventIdleBatch.Post(e);
 372         }
 373
 374
 375 /*
 376         //Post a Change message for renames containing the old and new names
 377         void OnRenameEvent(object sender, RenamedEventArgs e)
 378         {
 379             var oldFullPath = e.OldFullPath;
 380             var fullPath = e.FullPath;
 381             if (Ignore(oldFullPath) || Ignore(fullPath))
 382                 return;
 383
 384             _agent.Post(new WorkflowState
 385             {
 386                 AccountInfo=AccountInfo,
 387                 OldPath = oldFullPath,
 388                 OldFileName = e.OldName,
 389                 Path = fullPath,
 390                 FileName = e.Name,
 391                 TriggeringChange = e.ChangeType
 392             });
 393         }
 394 */
 395
 396         //Post a Change message for moves containing the old and new names
 397         void OnMoveEvent(object sender, MovedEventArgs e)
 398         {
 399             var oldFullPath = e.OldFullPath;
 400             var fullPath = e.FullPath;
 401             if (Ignore(oldFullPath) || Ignore(fullPath))
 402                 return;
 403
 404             _eventIdleBatch.Post(e);
 405         }
 406
 407
 408
 409         private Dictionary<WatcherChangeTypes, FileStatus> _statusDict = new Dictionary<WatcherChangeTypes, FileStatus>
 410                                                                              {
 411             {WatcherChangeTypes.Created,FileStatus.Created},
 412             {WatcherChangeTypes.Changed,FileStatus.Modified},
 413             {WatcherChangeTypes.Deleted,FileStatus.Deleted},
 414             {WatcherChangeTypes.Renamed,FileStatus.Renamed}
 415         };
 416
 417         private Dictionary<string, string> _ignoreFiles=new Dictionary<string, string>();
 418
 419         private WorkflowState UpdateFileStatus(WorkflowState state)
 420         {
 421             if (state==null)
 422                 throw new ArgumentNullException("state");
 423             if (String.IsNullOrWhiteSpace(state.Path))
 424                 throw new ArgumentException("The state's Path can't be empty","state");
 425             Contract.EndContractBlock();
 426
 427             var path = state.Path;
 428             var status = _statusDict[state.TriggeringChange];
 429             var oldStatus = Workflow.StatusKeeper.GetFileStatus(path);
 430             if (status == oldStatus)
 431             {
 432                 state.Status = status;
 433                 state.Skip = true;
 434                 return state;
 435             }
 436             if (state.Status == FileStatus.Renamed)
 437                 Workflow.ClearFileStatus(path);
 438
 439             state.Status = Workflow.SetFileStatus(path, status);
 440             return state;
 441         }
 442
 443         private WorkflowState UpdateOverlayStatus(WorkflowState state)
 444         {
 445             if (state==null)
 446                 throw new ArgumentNullException("state");
 447             Contract.EndContractBlock();
 448
 449             if (state.Skip)
 450                 return state;
 451
 452             switch (state.Status)
 453             {
 454                 case FileStatus.Created:
 455                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash);
 456                     break;
 457                 case FileStatus.Modified:
 458                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash);
 459                     break;
 460                 case FileStatus.Deleted:
 461                     //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
 462                     break;
 463                 case FileStatus.Renamed:
 464                     this.StatusKeeper.ClearFileStatus(state.OldPath);
 465                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash);
 466                     break;
 467                 case FileStatus.Unchanged:
 468                     this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal,state.ShortHash);
 469                     break;
 470             }
 471
 472             if (state.Status == FileStatus.Deleted)
 473                 NativeMethods.RaiseChangeNotification(Path.GetDirectoryName(state.Path));
 474             else
 475                 NativeMethods.RaiseChangeNotification(state.Path);
 476             return state;
 477         }
 478
 479
 480         private WorkflowState UpdateFileChecksum(WorkflowState state)
 481         {
 482             if (state.Skip)
 483                 return state;
 484
 485             if (state.Status == FileStatus.Deleted)
 486                 return state;
 487
 488             var path = state.Path;
 489             //Skip calculation for folders
 490             if (Directory.Exists(path))
 491                 return state;
 492
 493             var info = new FileInfo(path);
 494
 495             using (StatusNotification.GetNotifier("Hashing {0}", "Finished Hashing {0}", info.Name))
 496             {
 497
 498                 var shortHash = info.ComputeShortHash();
 499
 500                 string merkleHash = info.CalculateHash(StatusKeeper.BlockSize, StatusKeeper.BlockHash);
 501                 StatusKeeper.UpdateFileChecksum(path, shortHash, merkleHash);
 502
 503                 state.Hash = merkleHash;
 504                 return state;
 505             }
 506         }
 507
 508         //Does the file exist in the container's local folder?
 509         public bool Exists(string relativePath)
 510         {
 511             if (String.IsNullOrWhiteSpace(relativePath))
 512                 throw new ArgumentNullException("relativePath");
 513             //A RootPath must be set before calling this method
 514             if (String.IsNullOrWhiteSpace(RootPath))
 515                 throw new InvalidOperationException("RootPath was not set");
 516             Contract.EndContractBlock();
 517             //Create the absolute path by combining the RootPath with the relativePath
 518             var absolutePath=Path.Combine(RootPath, relativePath);
 519             //Is this a valid file?
 520             if (File.Exists(absolutePath))
 521                 return true;
 522             //Or a directory?
 523             if (Directory.Exists(absolutePath))
 524                 return true;
 525             //Fail if it is neither
 526             return false;
 527         }
 528
 529         public static FileAgent GetFileAgent(AccountInfo accountInfo)
 530         {
 531             return GetFileAgent(accountInfo.AccountPath);
 532         }
 533
 534         public static FileAgent GetFileAgent(string rootPath)
 535         {
 536             return AgentLocator<FileAgent>.Get(rootPath.ToLower());
 537         }
 538
 539
 540         public FileSystemInfo GetFileSystemInfo(string relativePath)
 541         {
 542             if (String.IsNullOrWhiteSpace(relativePath))
 543                 throw new ArgumentNullException("relativePath");
 544             //A RootPath must be set before calling this method
 545             if (String.IsNullOrWhiteSpace(RootPath))
 546                 throw new InvalidOperationException("RootPath was not set");
 547             Contract.EndContractBlock();
 548
 549             var absolutePath = Path.Combine(RootPath, relativePath);
 550
 551             if (Directory.Exists(absolutePath))
 552                 return new DirectoryInfo(absolutePath).WithProperCapitalization();
 553             else
 554                 return new FileInfo(absolutePath).WithProperCapitalization();
 555
 556         }
 557
 558         public void Delete(string relativePath)
 559         {
 560             var absolutePath = Path.Combine(RootPath, relativePath).ToLower();
 561             if (Log.IsDebugEnabled)
 562                 Log.DebugFormat("Deleting {0}", absolutePath);
 563             if (File.Exists(absolutePath))
 564             {
 565                 try
 566                 {
 567                     File.Delete(absolutePath);
 568                 }
 569                 //The file may have been deleted by another thread. Just ignore the relevant exception
 570                 catch (FileNotFoundException) { }
 571             }
 572             else if (Directory.Exists(absolutePath))
 573             {
 574                 try
 575                 {
 576                     Directory.Delete(absolutePath, true);
 577                 }
 578                 //The directory may have been deleted by another thread. Just ignore the relevant exception
 579                 catch (DirectoryNotFoundException){}
 580             }
 581
 582             //_ignoreFiles[absolutePath] = absolutePath;
 583             StatusKeeper.ClearFileStatus(absolutePath);
 584         }
 585     }
 586 }