code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/PithosMonitor.cs

   1 using System;
   2 using System.Collections.Concurrent;
   3 using System.Collections.Generic;
   4 using System.ComponentModel.Composition;
   5 using System.Diagnostics;
   6 using System.Diagnostics.Contracts;
   7 using System.IO;
   8 using System.Linq;
   9 using System.Net.NetworkInformation;
  10 using System.Security.Cryptography;
  11 using System.ServiceModel.Description;
  12 using System.Text;
  13 using System.Threading;
  14 using System.Threading.Tasks;
  15 using Castle.ActiveRecord.Queries;
  16 using Microsoft.WindowsAPICodePack.Net;
  17 using Pithos.Core.Agents;
  18 using Pithos.Interfaces;
  19 using System.ServiceModel;
  20 using Pithos.Network;
  21 using log4net;
  22
  23 namespace Pithos.Core
  24 {
  25     [Export(typeof(PithosMonitor))]
  26     public class PithosMonitor:IDisposable
  27     {
  28         private int _blockSize;
  29         private string _blockHash;
  30
  31         [Import]
  32         public IPithosSettings Settings{get;set;}
  33
  34         private IStatusKeeper _statusKeeper;
  35
  36         [Import]
  37         public IStatusKeeper StatusKeeper
  38         {
  39             get { return _statusKeeper; }
  40             set
  41             {
  42                 _statusKeeper = value;
  43                 FileAgent.StatusKeeper = value;
  44             }
  45         }
  46
  47
  48         private IPithosWorkflow _workflow;
  49
  50         [Import]
  51         public IPithosWorkflow Workflow
  52         {
  53             get { return _workflow; }
  54             set
  55             {
  56                 _workflow = value;
  57                 FileAgent.Workflow = value;
  58             }
  59         }
  60
  61         public ICloudClient CloudClient { get; set; }
  62
  63         public IStatusNotification StatusNotification { get; set; }
  64
  65         //[Import]
  66         public FileAgent FileAgent { get; private set; }
  67
  68         private WorkflowAgent _workflowAgent;
  69
  70         [Import]
  71         public WorkflowAgent WorkflowAgent
  72         {
  73             get { return _workflowAgent; }
  74             set
  75             {
  76                 _workflowAgent = value;
  77                 FileAgent.WorkflowAgent = value;
  78             }
  79         }
  80
  81         [Import]
  82         public NetworkAgent NetworkAgent { get; set; }
  83
  84         public string UserName { get; set; }
  85         private string _apiKey;
  86         public string ApiKey
  87         {
  88             get { return _apiKey; }
  89             set
  90             {
  91                 _apiKey = value;
  92                 if (_accountInfo != null)
  93                     _accountInfo.Token = value;
  94             }
  95         }
  96
  97         private AccountInfo _accountInfo;
  98
  99
 100
 101
 102         private static readonly ILog Log = LogManager.GetLogger(typeof(PithosMonitor));
 103
 104
 105         public bool Pause
 106         {
 107             get { return FileAgent.Pause; }
 108             set
 109             {
 110                 FileAgent.Pause = value;
 111                 if (value)
 112                 {
 113                     StatusKeeper.SetPithosStatus(PithosStatus.SyncPaused);
 114                     StatusNotification.NotifyChange("Paused");
 115                 }
 116                 else
 117                 {
 118                     StatusKeeper.SetPithosStatus(PithosStatus.InSynch);
 119                     StatusNotification.NotifyChange("Synchronizing");
 120                 }
 121             }
 122         }
 123
 124         private string _rootPath;
 125         public string RootPath
 126         {
 127             get { return _rootPath; }
 128             set
 129             {
 130                 _rootPath = String.IsNullOrWhiteSpace(value)
 131                     ? String.Empty
 132                     : value.ToLower();
 133             }
 134         }
 135
 136
 137         CancellationTokenSource _cancellationSource;
 138
 139         public PithosMonitor()
 140         {
 141             FileAgent = new FileAgent();
 142
 143         }
 144         private bool _started;
 145
 146         public void Start()
 147         {
 148             if (String.IsNullOrWhiteSpace(ApiKey))
 149                 throw new InvalidOperationException("The ApiKey is empty");
 150             if (String.IsNullOrWhiteSpace(UserName))
 151                 throw new InvalidOperationException("The UserName is empty");
 152             if (String.IsNullOrWhiteSpace(AuthenticationUrl))
 153                 throw new InvalidOperationException("The Authentication url is empty");
 154             Contract.EndContractBlock();
 155
 156             //If the account doesn't have a valid path, don't start monitoring but don't throw either
 157             if (String.IsNullOrWhiteSpace(RootPath))
 158                 //TODO; Warn user?
 159                 return;
 160
 161             StatusNotification.NotifyChange("Starting");
 162             if (_started)
 163             {
 164                 if (!_cancellationSource.IsCancellationRequested)
 165                     return;
 166             }
 167             _cancellationSource = new CancellationTokenSource();
 168
 169
 170             CloudClient=new CloudFilesClient(UserName,ApiKey);
 171             var proxyUri = ProxyFromSettings();
 172             CloudClient.Proxy = proxyUri;
 173             CloudClient.UsePithos = true;
 174             CloudClient.AuthenticationUrl = this.AuthenticationUrl;
 175
 176             _accountInfo = CloudClient.Authenticate();
 177             _accountInfo.SiteUri = AuthenticationUrl;
 178             _accountInfo.AccountPath = RootPath;
 179
 180
 181             var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
 182             if (!Directory.Exists(pithosFolder))
 183                 Directory.CreateDirectory(pithosFolder);
 184             //Create the cache folder and ensure it is hidden
 185             CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
 186
 187             var policy=CloudClient.GetAccountPolicies(_accountInfo);
 188
 189             StatusNotification.NotifyAccount(policy);
 190             EnsurePithosContainers();
 191
 192             StatusKeeper.BlockHash = _blockHash;
 193             StatusKeeper.BlockSize = _blockSize;
 194
 195             StatusKeeper.StartProcessing(_cancellationSource.Token);
 196             IndexLocalFiles();
 197             StartWatcherAgent();
 198
 199             StartNetworkAgent();
 200
 201             StartWorkflowAgent();
 202             WorkflowAgent.RestartInterruptedFiles(_accountInfo);
 203             _started = true;
 204         }
 205
 206         private void EnsurePithosContainers()
 207         {
 208
 209             //Create the two default containers if they are missing
 210             var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
 211             foreach (var container in pithosContainers)
 212             {
 213                 var info=CloudClient.GetContainerInfo(this.UserName, container);
 214                 if (info == ContainerInfo.Empty)
 215                 {
 216                     CloudClient.CreateContainer(this.UserName, container);
 217                     info = CloudClient.GetContainerInfo(this.UserName, container);
 218                 }
 219                 _blockSize = info.BlockSize;
 220                 _blockHash = info.BlockHash;
 221                 _accountInfo.BlockSize = _blockSize;
 222                 _accountInfo.BlockHash = _blockHash;
 223             }
 224         }
 225
 226         public string AuthenticationUrl { get; set; }
 227
 228         private Uri ProxyFromSettings()
 229         {
 230             if (Settings.UseManualProxy)
 231             {
 232                 var proxyUri = new UriBuilder
 233                                    {
 234                                        Host = Settings.ProxyServer,
 235                                        Port = Settings.ProxyPort
 236                                    };
 237                 if (Settings.ProxyAuthentication)
 238                 {
 239                     proxyUri.UserName = Settings.ProxyUsername;
 240                     proxyUri.Password = Settings.ProxyPassword;
 241                 }
 242                 return proxyUri.Uri;
 243             }
 244             return null;
 245         }
 246
 247         private void IndexLocalFiles()
 248         {
 249             StatusNotification.NotifyChange("Indexing Local Files",TraceLevel.Info);
 250             using (log4net.ThreadContext.Stacks["Monitor"].Push("Indexing local files"))
 251             {
 252                 Log.Info("START");
 253                 try
 254                 {
 255                     var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
 256                     var directory = new DirectoryInfo(RootPath);
 257                     var files =
 258                         from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
 259                         where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
 260                               !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
 261                         select file;
 262                     StatusKeeper.ProcessExistingFiles(files);
 263
 264                 }
 265                 catch (Exception exc)
 266                 {
 267                     Log.Error("[ERROR]", exc);
 268                 }
 269                 finally
 270                 {
 271                     Log.Info("[END]");
 272                 }
 273             }
 274         }
 275
 276
 277
 278
 279
 280         private void StartWorkflowAgent()
 281         {
 282             //On Vista and up we can check for a network connection
 283             bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
 284             //If we are not connected retry later
 285             if (!connected)
 286             {
 287                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
 288                 return;
 289             }
 290
 291             try
 292             {
 293                 WorkflowAgent.StatusNotification = StatusNotification;
 294                 WorkflowAgent.Start();
 295             }
 296             catch (Exception)
 297             {
 298                 //Faild to authenticate due to network or account error
 299                 //Retry after a while
 300                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
 301             }
 302         }
 303
 304         internal class LocalFileComparer:EqualityComparer<CloudAction>
 305         {
 306             public override bool Equals(CloudAction x, CloudAction y)
 307             {
 308                 if (x.Action != y.Action)
 309                     return false;
 310                 if (x.LocalFile != null && y.LocalFile != null && !x.LocalFile.FullName.Equals(y.LocalFile.FullName))
 311                     return false;
 312                 if (x.CloudFile != null && y.CloudFile != null )
 313                 {
 314                     if (x.CloudFile.Hash == null & y.CloudFile.Hash != null)
 315                         return false;
 316                     if (x.CloudFile.Hash != null & y.CloudFile.Hash == null)
 317                         return false;
 318                     if (x.CloudFile.Hash == null & y.CloudFile.Hash == null)
 319                         return (x.CloudFile.Name == y.CloudFile.Name);
 320                     if (!x.CloudFile.Hash.Equals(y.CloudFile.Hash))
 321                         return false;
 322                 }
 323                 if (x.CloudFile == null ^ y.CloudFile == null ||
 324                     x.LocalFile == null ^ y.LocalFile == null)
 325                     return false;
 326                 return true;
 327             }
 328
 329             public override int GetHashCode(CloudAction obj)
 330             {
 331                 var hash1 = (obj.LocalFile == null) ? int.MaxValue : obj.LocalFile.FullName.GetHashCode();
 332                 var hash2 = (obj.CloudFile == null) ? int.MaxValue : (obj.CloudFile.Hash ?? obj.CloudFile.Name??"").GetHashCode();
 333                 var hash3 = obj.Action.GetHashCode();
 334                 return hash1 ^ hash2 & hash3;
 335             }
 336         }
 337
 338         private Timer timer;
 339
 340         private void StartNetworkAgent()
 341         {
 342
 343             NetworkAgent.AddAccount(_accountInfo);
 344
 345             NetworkAgent.StatusNotification = StatusNotification;
 346
 347             NetworkAgent.Start();
 348
 349             NetworkAgent.PollRemoteFiles();
 350         }
 351
 352         //Make sure a hidden cache folder exists to store partial downloads
 353         private static string CreateHiddenFolder(string rootPath, string folderName)
 354         {
 355             if (String.IsNullOrWhiteSpace(rootPath))
 356                 throw new ArgumentNullException("rootPath");
 357             if (!Path.IsPathRooted(rootPath))
 358                 throw new ArgumentException("rootPath");
 359             if (String.IsNullOrWhiteSpace(folderName))
 360                 throw new ArgumentNullException("folderName");
 361             Contract.EndContractBlock();
 362
 363             var folder = Path.Combine(rootPath, folderName);
 364             if (!Directory.Exists(folder))
 365             {
 366                 var info = Directory.CreateDirectory(folder);
 367                 info.Attributes |= FileAttributes.Hidden;
 368
 369                 Log.InfoFormat("Created cache Folder: {0}", folder);
 370             }
 371             else
 372             {
 373                 var info = new DirectoryInfo(folder);
 374                 if ((info.Attributes & FileAttributes.Hidden) == 0)
 375                 {
 376                     info.Attributes |= FileAttributes.Hidden;
 377                     Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
 378                 }
 379             }
 380             return folder;
 381         }
 382
 383
 384
 385
 386         private void StartWatcherAgent()
 387         {
 388             AgentLocator<FileAgent>.Register(FileAgent,RootPath);
 389
 390             FileAgent.StatusKeeper = StatusKeeper;
 391             FileAgent.Workflow = Workflow;
 392             FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
 393             FileAgent.Start(_accountInfo, RootPath);
 394         }
 395
 396         public void Stop()
 397         {
 398             AgentLocator<FileAgent>.Remove(RootPath);
 399
 400             if (FileAgent!=null)
 401                 FileAgent.Stop();
 402             FileAgent = null;
 403             if (timer != null)
 404                 timer.Dispose();
 405             timer = null;
 406         }
 407
 408
 409         ~PithosMonitor()
 410         {
 411             Dispose(false);
 412         }
 413
 414         public void Dispose()
 415         {
 416             Dispose(true);
 417             GC.SuppressFinalize(this);
 418         }
 419
 420         protected virtual void Dispose(bool disposing)
 421         {
 422             if (disposing)
 423             {
 424                 Stop();
 425             }
 426         }
 427
 428
 429         public void MoveFileStates(string oldPath, string newPath)
 430         {
 431             if (String.IsNullOrWhiteSpace(oldPath))
 432                 throw new ArgumentNullException("oldPath");
 433             if (!Path.IsPathRooted(oldPath))
 434                 throw new ArgumentException("oldPath must be an absolute path","oldPath");
 435             if (string.IsNullOrWhiteSpace(newPath))
 436                 throw new ArgumentNullException("newPath");
 437             if (!Path.IsPathRooted(newPath))
 438                 throw new ArgumentException("newPath must be an absolute path","newPath");
 439             Contract.EndContractBlock();
 440
 441             StatusKeeper.ChangeRoots(oldPath, newPath);
 442         }
 443
 444         public void AddSelectivePaths(string[] added)
 445         {
 446            /* FileAgent.SelectivePaths.AddRange(added);
 447             NetworkAgent.SyncPaths(added);*/
 448         }
 449
 450         public void RemoveSelectivePaths(string[] removed)
 451         {
 452             FileAgent.SelectivePaths.RemoveAll(removed.Contains);
 453             foreach (var removedPath in removed.Where(Directory.Exists))
 454             {
 455                 Directory.Delete(removedPath,true);
 456             }
 457         }
 458
 459         public IEnumerable<string> GetRootFolders()
 460         {
 461             var dirs = from container in CloudClient.ListContainers(UserName)
 462                        from dir in CloudClient.ListObjects(UserName, container.Name, "")
 463                        select dir.Name;
 464             return dirs;
 465         }
 466
 467         public ObjectInfo GetObjectInfo(string filePath)
 468         {
 469             if (String.IsNullOrWhiteSpace(filePath))
 470                 throw new ArgumentNullException("filePath");
 471             Contract.EndContractBlock();
 472
 473             var file=new FileInfo(filePath);
 474             string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
 475             var relativePath = file.AsRelativeTo(RootPath);
 476
 477             string accountName,container;
 478
 479             var parts=relativePath.Split('\\');
 480
 481             var accountInfo = _accountInfo;
 482             if (relativePath.StartsWith(FolderConstants.OthersFolder))
 483             {
 484                 accountName = parts[1];
 485                 container = parts[2];
 486                 relativeUrl = String.Join("/", parts.Splice(3));
 487                 //Create the root URL for the target account
 488                 var oldName = UserName;
 489                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
 490                 var nameIndex=absoluteUri.IndexOf(oldName);
 491                 var root=absoluteUri.Substring(0, nameIndex);
 492
 493                 accountInfo = new AccountInfo
 494                 {
 495                     UserName = accountName,
 496                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 497                     StorageUri = new Uri(root + accountName),
 498                     BlockHash=accountInfo.BlockHash,
 499                     BlockSize=accountInfo.BlockSize,
 500                     Token=accountInfo.Token
 501                 };
 502             }
 503             else
 504             {
 505                 accountName = this.UserName;
 506                 container = parts[0];
 507                 relativeUrl = String.Join("/", parts.Splice(1));
 508             }
 509
 510             var client = new CloudFilesClient(accountInfo);
 511             var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
 512             return objectInfo;
 513         }
 514
 515         public Task<ContainerInfo> GetContainerInfo(string filePath)
 516         {
 517             if (String.IsNullOrWhiteSpace(filePath))
 518                 throw new ArgumentNullException("filePath");
 519             Contract.EndContractBlock();
 520
 521             var file=new FileInfo(filePath);
 522             var relativePath = file.AsRelativeTo(RootPath);
 523
 524             string accountName,container;
 525
 526             var parts=relativePath.Split('\\');
 527
 528             var accountInfo = _accountInfo;
 529             if (relativePath.StartsWith(FolderConstants.OthersFolder))
 530             {
 531                 accountName = parts[1];
 532                 container = parts[2];
 533                 //Create the root URL for the target account
 534                 var oldName = UserName;
 535                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
 536                 var nameIndex=absoluteUri.IndexOf(oldName);
 537                 var root=absoluteUri.Substring(0, nameIndex);
 538
 539                 accountInfo = new AccountInfo
 540                 {
 541                     UserName = accountName,
 542                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 543                     StorageUri = new Uri(root + accountName),
 544                     BlockHash=accountInfo.BlockHash,
 545                     BlockSize=accountInfo.BlockSize,
 546                     Token=accountInfo.Token
 547                 };
 548             }
 549             else
 550             {
 551                 accountName = UserName;
 552                 container = parts[0];
 553             }
 554
 555             return Task.Factory.StartNew(() =>
 556             {
 557                 var client = new CloudFilesClient(accountInfo);
 558                 var containerInfo = client.GetContainerInfo(accountName, container);
 559                 return containerInfo;
 560             });
 561         }
 562     }
 563
 564
 565     public interface IStatusNotification
 566     {
 567         void NotifyChange(string status,TraceLevel level=TraceLevel.Info);
 568         void NotifyChangedFile(string filePath);
 569         void NotifyAccount(AccountInfo policy);
 570     }
 571 }