code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/PithosMonitor.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="PithosMonitor.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System;
  43 using System.Collections.Concurrent;
  44 using System.Collections.Generic;
  45 using System.ComponentModel.Composition;
  46 using System.Diagnostics;
  47 using System.Diagnostics.Contracts;
  48 using System.IO;
  49 using System.Linq;
  50 using System.Net;
  51 using System.Net.NetworkInformation;
  52 using System.Security.Cryptography;
  53 using System.ServiceModel.Description;
  54 using System.Text;
  55 using System.Threading;
  56 using System.Threading.Tasks;
  57 using Castle.ActiveRecord.Queries;
  58 using Microsoft.WindowsAPICodePack.Net;
  59 using Pithos.Core.Agents;
  60 using Pithos.Interfaces;
  61 using System.ServiceModel;
  62 using Pithos.Network;
  63 using log4net;
  64
  65 namespace Pithos.Core
  66 {
  67     [Export(typeof(PithosMonitor))]
  68     public class PithosMonitor:IDisposable
  69     {
  70         private int _blockSize;
  71         private string _blockHash;
  72
  73         [Import]
  74         public IPithosSettings Settings{get;set;}
  75
  76         private IStatusKeeper _statusKeeper;
  77
  78         [Import]
  79         public IStatusKeeper StatusKeeper
  80         {
  81             get { return _statusKeeper; }
  82             set
  83             {
  84                 _statusKeeper = value;
  85                 FileAgent.StatusKeeper = value;
  86             }
  87         }
  88
  89
  90         private IPithosWorkflow _workflow;
  91
  92         [Import]
  93         public IPithosWorkflow Workflow
  94         {
  95             get { return _workflow; }
  96             set
  97             {
  98                 _workflow = value;
  99                 FileAgent.Workflow = value;
 100             }
 101         }
 102
 103         public ICloudClient CloudClient { get; set; }
 104
 105         public IStatusNotification StatusNotification { get; set; }
 106
 107         //[Import]
 108         public FileAgent FileAgent { get; private set; }
 109
 110         private WorkflowAgent _workflowAgent;
 111
 112         [Import]
 113         public WorkflowAgent WorkflowAgent
 114         {
 115             get { return _workflowAgent; }
 116             set
 117             {
 118                 _workflowAgent = value;
 119                 FileAgent.WorkflowAgent = value;
 120             }
 121         }
 122
 123         [Import]
 124         public NetworkAgent NetworkAgent { get; set; }
 125         [Import]
 126         public PollAgent PollAgent { get; set; }
 127
 128         public string UserName { get; set; }
 129         private string _apiKey;
 130         public string ApiKey
 131         {
 132             get { return _apiKey; }
 133             set
 134             {
 135                 _apiKey = value;
 136                 if (_accountInfo != null)
 137                     _accountInfo.Token = value;
 138             }
 139         }
 140
 141         private AccountInfo _accountInfo;
 142
 143
 144
 145
 146         private static readonly ILog Log = LogManager.GetLogger(typeof(PithosMonitor));
 147
 148
 149         public bool Pause
 150         {
 151             get { return FileAgent.Pause; }
 152             set
 153             {
 154                 FileAgent.Pause = value;
 155                 if (value)
 156                 {
 157                     StatusKeeper.SetPithosStatus(PithosStatus.SyncPaused);
 158                     StatusNotification.NotifyChange("Paused");
 159                 }
 160                 else
 161                 {
 162                     StatusKeeper.SetPithosStatus(PithosStatus.InSynch);
 163                     StatusNotification.NotifyChange("Synchronizing");
 164                 }
 165             }
 166         }
 167
 168         private string _rootPath;
 169         public string RootPath
 170         {
 171             get { return _rootPath; }
 172             set
 173             {
 174                 _rootPath = String.IsNullOrWhiteSpace(value)
 175                     ? String.Empty
 176                     : value.ToLower();
 177             }
 178         }
 179
 180
 181         CancellationTokenSource _cancellationSource;
 182
 183         public PithosMonitor()
 184         {
 185             FileAgent = new FileAgent();
 186
 187         }
 188         private bool _started;
 189
 190         public void Start()
 191         {
 192             if (String.IsNullOrWhiteSpace(ApiKey))
 193                 throw new InvalidOperationException("The ApiKey is empty");
 194             if (String.IsNullOrWhiteSpace(UserName))
 195                 throw new InvalidOperationException("The UserName is empty");
 196             if (String.IsNullOrWhiteSpace(AuthenticationUrl))
 197                 throw new InvalidOperationException("The Authentication url is empty");
 198             Contract.EndContractBlock();
 199
 200             //If the account doesn't have a valid path, don't start monitoring but don't throw either
 201             if (String.IsNullOrWhiteSpace(RootPath))
 202                 //TODO; Warn user?
 203                 return;
 204
 205             WorkflowAgent.StatusNotification = StatusNotification;
 206
 207             StatusNotification.NotifyChange("Starting");
 208             if (_started)
 209             {
 210                 if (!_cancellationSource.IsCancellationRequested)
 211                     return;
 212             }
 213             _cancellationSource = new CancellationTokenSource();
 214
 215
 216             CloudClient=new CloudFilesClient(UserName,ApiKey);
 217             CloudClient.UsePithos = true;
 218             CloudClient.AuthenticationUrl = this.AuthenticationUrl;
 219
 220             _accountInfo = CloudClient.Authenticate();
 221             _accountInfo.SiteUri = AuthenticationUrl;
 222             _accountInfo.AccountPath = RootPath;
 223
 224
 225             var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
 226             if (!Directory.Exists(pithosFolder))
 227                 Directory.CreateDirectory(pithosFolder);
 228             //Create the cache folder and ensure it is hidden
 229             CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
 230
 231             var policy=CloudClient.GetAccountPolicies(_accountInfo);
 232
 233             StatusNotification.NotifyAccount(policy);
 234             EnsurePithosContainers();
 235
 236             StatusKeeper.BlockHash = _blockHash;
 237             StatusKeeper.BlockSize = _blockSize;
 238
 239             StatusKeeper.StartProcessing(_cancellationSource.Token);
 240             IndexLocalFiles();
 241             //Extract the URIs from the string collection
 242             var settings = Settings.Accounts.First(s => s.AccountName == _accountInfo.UserName);
 243             var selectiveUrls=settings.SelectiveFolders.Cast<string>().Select(url => new Uri(url)).ToArray();
 244
 245             SetSelectivePaths(selectiveUrls,null,null);
 246
 247             StartWatcherAgent();
 248
 249             StartNetworkAgent();
 250
 251             WorkflowAgent.RestartInterruptedFiles(_accountInfo);
 252             _started = true;
 253         }
 254
 255         private void EnsurePithosContainers()
 256         {
 257
 258             //Create the two default containers if they are missing
 259             var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
 260             foreach (var container in pithosContainers)
 261             {
 262                 var info=CloudClient.GetContainerInfo(this.UserName, container);
 263                 if (info == ContainerInfo.Empty)
 264                 {
 265                     CloudClient.CreateContainer(this.UserName, container);
 266                     info = CloudClient.GetContainerInfo(this.UserName, container);
 267                 }
 268                 _blockSize = info.BlockSize;
 269                 _blockHash = info.BlockHash;
 270                 _accountInfo.BlockSize = _blockSize;
 271                 _accountInfo.BlockHash = _blockHash;
 272             }
 273         }
 274
 275         public string AuthenticationUrl { get; set; }
 276
 277         private void IndexLocalFiles()
 278         {
 279             StatusNotification.NotifyChange("Indexing Local Files",TraceLevel.Info);
 280             using (log4net.ThreadContext.Stacks["Monitor"].Push("Indexing local files"))
 281             {
 282                 Log.Info("START");
 283                 try
 284                 {
 285                     var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
 286                     var directory = new DirectoryInfo(RootPath);
 287                     var files =
 288                         from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
 289                         where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
 290                               !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
 291                         select file;
 292                     StatusKeeper.ProcessExistingFiles(files);
 293
 294                 }
 295                 catch (Exception exc)
 296                 {
 297                     Log.Error("[ERROR]", exc);
 298                 }
 299                 finally
 300                 {
 301                     Log.Info("[END]");
 302                 }
 303             }
 304         }
 305
 306
 307
 308
 309
 310        /* private void StartWorkflowAgent()
 311         {
 312             WorkflowAgent.StatusNotification = StatusNotification;
 313
 314 /*            //On Vista and up we can check for a network connection
 315             bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
 316             //If we are not connected retry later
 317             if (!connected)
 318             {
 319                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
 320                 return;
 321             }#1#
 322
 323             try
 324             {
 325                 WorkflowAgent.Start();
 326             }
 327             catch (Exception)
 328             {
 329                 //Faild to authenticate due to network or account error
 330                 //Retry after a while
 331                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
 332             }
 333         }*/
 334
 335         internal class LocalFileComparer:EqualityComparer<CloudAction>
 336         {
 337             public override bool Equals(CloudAction x, CloudAction y)
 338             {
 339                 if (x.Action != y.Action)
 340                     return false;
 341                 if (x.LocalFile != null && y.LocalFile != null && !x.LocalFile.FullName.Equals(y.LocalFile.FullName))
 342                     return false;
 343                 if (x.CloudFile != null && y.CloudFile != null )
 344                 {
 345                     if (x.CloudFile.Hash == null & y.CloudFile.Hash != null)
 346                         return false;
 347                     if (x.CloudFile.Hash != null & y.CloudFile.Hash == null)
 348                         return false;
 349                     if (x.CloudFile.Hash == null & y.CloudFile.Hash == null)
 350                         return (x.CloudFile.Name == y.CloudFile.Name);
 351                     if (!x.CloudFile.Hash.Equals(y.CloudFile.Hash))
 352                         return false;
 353                 }
 354                 if (x.CloudFile == null ^ y.CloudFile == null ||
 355                     x.LocalFile == null ^ y.LocalFile == null)
 356                     return false;
 357                 return true;
 358             }
 359
 360             public override int GetHashCode(CloudAction obj)
 361             {
 362                 var hash1 = (obj.LocalFile == null) ? int.MaxValue : obj.LocalFile.FullName.GetHashCode();
 363                 var hash2 = (obj.CloudFile == null) ? int.MaxValue : (obj.CloudFile.Hash ?? obj.CloudFile.Name??"").GetHashCode();
 364                 var hash3 = obj.Action.GetHashCode();
 365                 return hash1 ^ hash2 & hash3;
 366             }
 367         }
 368
 369         private Timer timer;
 370
 371         private void StartNetworkAgent()
 372         {
 373
 374             NetworkAgent.AddAccount(_accountInfo);
 375
 376             NetworkAgent.StatusNotification = StatusNotification;
 377
 378             NetworkAgent.Start();
 379
 380             PollAgent.AddAccount(_accountInfo);
 381
 382             PollAgent.StatusNotification = StatusNotification;
 383
 384             PollAgent.PollRemoteFiles();
 385         }
 386
 387         //Make sure a hidden cache folder exists to store partial downloads
 388         private static string CreateHiddenFolder(string rootPath, string folderName)
 389         {
 390             if (String.IsNullOrWhiteSpace(rootPath))
 391                 throw new ArgumentNullException("rootPath");
 392             if (!Path.IsPathRooted(rootPath))
 393                 throw new ArgumentException("rootPath");
 394             if (String.IsNullOrWhiteSpace(folderName))
 395                 throw new ArgumentNullException("folderName");
 396             Contract.EndContractBlock();
 397
 398             var folder = Path.Combine(rootPath, folderName);
 399             if (!Directory.Exists(folder))
 400             {
 401                 var info = Directory.CreateDirectory(folder);
 402                 info.Attributes |= FileAttributes.Hidden;
 403
 404                 Log.InfoFormat("Created cache Folder: {0}", folder);
 405             }
 406             else
 407             {
 408                 var info = new DirectoryInfo(folder);
 409                 if ((info.Attributes & FileAttributes.Hidden) == 0)
 410                 {
 411                     info.Attributes |= FileAttributes.Hidden;
 412                     Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
 413                 }
 414             }
 415             return folder;
 416         }
 417
 418
 419
 420
 421         private void StartWatcherAgent()
 422         {
 423             AgentLocator<FileAgent>.Register(FileAgent,RootPath);
 424
 425             FileAgent.StatusKeeper = StatusKeeper;
 426             FileAgent.Workflow = Workflow;
 427             FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
 428             FileAgent.Start(_accountInfo, RootPath);
 429         }
 430
 431         public void Stop()
 432         {
 433             AgentLocator<FileAgent>.Remove(RootPath);
 434
 435             if (FileAgent!=null)
 436                 FileAgent.Stop();
 437             FileAgent = null;
 438             if (timer != null)
 439                 timer.Dispose();
 440             timer = null;
 441         }
 442
 443
 444         ~PithosMonitor()
 445         {
 446             Dispose(false);
 447         }
 448
 449         public void Dispose()
 450         {
 451             Dispose(true);
 452             GC.SuppressFinalize(this);
 453         }
 454
 455         protected virtual void Dispose(bool disposing)
 456         {
 457             if (disposing)
 458             {
 459                 Stop();
 460             }
 461         }
 462
 463
 464         public void MoveFileStates(string oldPath, string newPath)
 465         {
 466             if (String.IsNullOrWhiteSpace(oldPath))
 467                 throw new ArgumentNullException("oldPath");
 468             if (!Path.IsPathRooted(oldPath))
 469                 throw new ArgumentException("oldPath must be an absolute path","oldPath");
 470             if (string.IsNullOrWhiteSpace(newPath))
 471                 throw new ArgumentNullException("newPath");
 472             if (!Path.IsPathRooted(newPath))
 473                 throw new ArgumentException("newPath must be an absolute path","newPath");
 474             Contract.EndContractBlock();
 475
 476             StatusKeeper.ChangeRoots(oldPath, newPath);
 477         }
 478
 479         public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed)
 480         {
 481             //Convert the uris to paths
 482             var selectivePaths = UrisToFilePaths(uris);
 483
 484             FileAgent.SelectivePaths=selectivePaths;
 485             PollAgent.SetSyncUris(uris);
 486
 487             var removedPaths = UrisToFilePaths(removed);
 488             UnversionSelectivePaths(removedPaths);
 489
 490         }
 491
 492         /// <summary>
 493         /// Mark all unselected paths as Unversioned
 494         /// </summary>
 495         /// <param name="removed"></param>
 496         private void UnversionSelectivePaths(List<string> removed)
 497         {
 498             if (removed == null)
 499                 return;
 500
 501             //Ensure we remove any file state below the deleted folders
 502             FileState.UnversionPaths(removed);
 503         }
 504
 505
 506         /// <summary>
 507         /// Return a list of absolute filepaths from a list of Uris
 508         /// </summary>
 509         /// <param name="uris"></param>
 510         /// <returns></returns>
 511         private List<string> UrisToFilePaths(IEnumerable<Uri> uris)
 512         {
 513             if (uris == null)
 514                 return new List<string>();
 515
 516             return (from uri in uris
 517                     let relativePath = _accountInfo.StorageUri
 518                         .MakeRelativeUri(uri)
 519                         .RelativeUriToFilePath()
 520                         //Trim the account name
 521                     select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList();
 522         }
 523
 524
 525         public ObjectInfo GetObjectInfo(string filePath)
 526         {
 527             if (String.IsNullOrWhiteSpace(filePath))
 528                 throw new ArgumentNullException("filePath");
 529             Contract.EndContractBlock();
 530
 531             var file=new FileInfo(filePath);
 532             string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
 533             var relativePath = file.AsRelativeTo(RootPath);
 534
 535             string accountName,container;
 536
 537             var parts=relativePath.Split('\\');
 538
 539             var accountInfo = _accountInfo;
 540             if (relativePath.StartsWith(FolderConstants.OthersFolder))
 541             {
 542                 accountName = parts[1];
 543                 container = parts[2];
 544                 relativeUrl = String.Join("/", parts.Splice(3));
 545                 //Create the root URL for the target account
 546                 var oldName = UserName;
 547                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
 548                 var nameIndex=absoluteUri.IndexOf(oldName);
 549                 var root=absoluteUri.Substring(0, nameIndex);
 550
 551                 accountInfo = new AccountInfo
 552                 {
 553                     UserName = accountName,
 554                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 555                     StorageUri = new Uri(root + accountName),
 556                     BlockHash=accountInfo.BlockHash,
 557                     BlockSize=accountInfo.BlockSize,
 558                     Token=accountInfo.Token
 559                 };
 560             }
 561             else
 562             {
 563                 accountName = this.UserName;
 564                 container = parts[0];
 565                 relativeUrl = String.Join("/", parts.Splice(1));
 566             }
 567
 568             var client = new CloudFilesClient(accountInfo);
 569             var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
 570             return objectInfo;
 571         }
 572
 573         public Task<ContainerInfo> GetContainerInfo(string filePath)
 574         {
 575             if (String.IsNullOrWhiteSpace(filePath))
 576                 throw new ArgumentNullException("filePath");
 577             Contract.EndContractBlock();
 578
 579             var file=new FileInfo(filePath);
 580             var relativePath = file.AsRelativeTo(RootPath);
 581
 582             string accountName,container;
 583
 584             var parts=relativePath.Split('\\');
 585
 586             var accountInfo = _accountInfo;
 587             if (relativePath.StartsWith(FolderConstants.OthersFolder))
 588             {
 589                 accountName = parts[1];
 590                 container = parts[2];
 591                 //Create the root URL for the target account
 592                 var oldName = UserName;
 593                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
 594                 var nameIndex=absoluteUri.IndexOf(oldName);
 595                 var root=absoluteUri.Substring(0, nameIndex);
 596
 597                 accountInfo = new AccountInfo
 598                 {
 599                     UserName = accountName,
 600                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 601                     StorageUri = new Uri(root + accountName),
 602                     BlockHash=accountInfo.BlockHash,
 603                     BlockSize=accountInfo.BlockSize,
 604                     Token=accountInfo.Token
 605                 };
 606             }
 607             else
 608             {
 609                 accountName = UserName;
 610                 container = parts[0];
 611             }
 612
 613             return Task.Factory.StartNew(() =>
 614             {
 615                 var client = new CloudFilesClient(accountInfo);
 616                 var containerInfo = client.GetContainerInfo(accountName, container);
 617                 return containerInfo;
 618             });
 619         }
 620     }
 621 }