code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/PithosMonitor.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="PithosMonitor.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System;
  43 using System.Collections.Generic;
  44 using System.ComponentModel.Composition;
  45 using System.Diagnostics.Contracts;
  46 using System.IO;
  47 using System.Linq;
  48 using System.Reflection;
  49 using System.Threading;
  50 using System.Threading.Tasks;
  51 using Pithos.Core.Agents;
  52 using Pithos.Interfaces;
  53 using Pithos.Network;
  54 using log4net;
  55
  56 namespace Pithos.Core
  57 {
  58     [Export(typeof(PithosMonitor))]
  59     public class PithosMonitor:IDisposable
  60     {
  61         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  62
  63         private int _blockSize;
  64         private string _blockHash;
  65
  66         [Import]
  67         public IPithosSettings Settings{get;set;}
  68
  69         private IStatusKeeper _statusKeeper;
  70
  71         [Import]
  72         public IStatusKeeper StatusKeeper
  73         {
  74             get { return _statusKeeper; }
  75             set
  76             {
  77                 _statusKeeper = value;
  78                 FileAgent.StatusKeeper = value;
  79             }
  80         }
  81
  82
  83
  84
  85         private IPithosWorkflow _workflow;
  86
  87         [Import]
  88         public IPithosWorkflow Workflow
  89         {
  90             get { return _workflow; }
  91             set
  92             {
  93                 _workflow = value;
  94                 FileAgent.Workflow = value;
  95             }
  96         }
  97
  98         public ICloudClient CloudClient { get; set; }
  99
 100         public IStatusNotification StatusNotification { get; set; }
 101
 102         //[Import]
 103         public FileAgent FileAgent { get; private set; }
 104
 105         private WorkflowAgent _workflowAgent;
 106
 107         [Import]
 108         public WorkflowAgent WorkflowAgent
 109         {
 110             get { return _workflowAgent; }
 111             set
 112             {
 113                 _workflowAgent = value;
 114                 FileAgent.WorkflowAgent = value;
 115             }
 116         }
 117
 118         [Import]
 119         public NetworkAgent NetworkAgent { get; set; }
 120         [Import]
 121         public PollAgent PollAgent { get; set; }
 122
 123         private Selectives _selectives;
 124
 125         [Import]
 126         public Selectives Selectives
 127         {
 128             get { return _selectives; }
 129             set
 130             {
 131                 _selectives = value;
 132                 FileAgent.Selectives = value;
 133             }
 134         }
 135
 136         public string UserName { get; set; }
 137         private string _apiKey;
 138         public string ApiKey
 139         {
 140             get { return _apiKey; }
 141             set
 142             {
 143                 _apiKey = value;
 144                 if (_accountInfo != null)
 145                     _accountInfo.Token = value;
 146             }
 147         }
 148
 149         private AccountInfo _accountInfo;
 150
 151         public AccountInfo Account
 152         {
 153             get { return _accountInfo; }
 154         }
 155
 156
 157
 158
 159
 160
 161         public bool Pause
 162         {
 163             get { return FileAgent.Pause; }
 164             set
 165             {
 166                 FileAgent.Pause = value;
 167             }
 168         }
 169
 170         private string _rootPath;
 171         public string RootPath
 172         {
 173             get { return _rootPath; }
 174             set
 175             {
 176                 _rootPath = String.IsNullOrWhiteSpace(value)
 177                     ? String.Empty
 178                     : value.ToLower();
 179             }
 180         }
 181
 182
 183         CancellationTokenSource _cancellationSource;
 184
 185         public PithosMonitor()
 186         {
 187             FileAgent = new FileAgent();
 188         }
 189         private bool _started;
 190
 191         public void Start()
 192         {
 193             if (String.IsNullOrWhiteSpace(ApiKey))
 194                 throw new InvalidOperationException("The ApiKey is empty");
 195             if (String.IsNullOrWhiteSpace(UserName))
 196                 throw new InvalidOperationException("The UserName is empty");
 197             if (String.IsNullOrWhiteSpace(AuthenticationUrl))
 198                 throw new InvalidOperationException("The Authentication url is empty");
 199             Contract.EndContractBlock();
 200
 201             //If the account doesn't have a valid path, don't start monitoring but don't throw either
 202             if (String.IsNullOrWhiteSpace(RootPath))
 203                 //TODO; Warn user?
 204                 return;
 205
 206             WorkflowAgent.StatusNotification = StatusNotification;
 207
 208             StatusNotification.NotifyChange("Starting");
 209             if (_started)
 210             {
 211                 if (!_cancellationSource.IsCancellationRequested)
 212                     return;
 213             }
 214             _cancellationSource = new CancellationTokenSource();
 215
 216             lock (this)
 217             {
 218                 CloudClient = new CloudFilesClient(UserName, ApiKey)
 219                                   {UsePithos = true, AuthenticationUrl = AuthenticationUrl};
 220                 _accountInfo = CloudClient.Authenticate();
 221             }
 222             _accountInfo.SiteUri = AuthenticationUrl;
 223             _accountInfo.AccountPath = RootPath;
 224
 225
 226             var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
 227             if (!Directory.Exists(pithosFolder))
 228                 Directory.CreateDirectory(pithosFolder);
 229             //Create the cache folder and ensure it is hidden
 230             CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
 231
 232             var policy=CloudClient.GetAccountPolicies(_accountInfo);
 233
 234             StatusNotification.NotifyAccount(policy);
 235             EnsurePithosContainers();
 236
 237             StatusKeeper.BlockHash = _blockHash;
 238             StatusKeeper.BlockSize = _blockSize;
 239
 240             StatusKeeper.StartProcessing(_cancellationSource.Token);
 241             IndexLocalFiles();
 242             //Extract the URIs from the string collection
 243             var settings = Settings.Accounts.First(s => s.AccountKey == _accountInfo.AccountKey );
 244
 245             var selectiveUrls=settings.SelectiveFolders.Cast<string>().Select(url => new Uri(url,UriKind.RelativeOrAbsolute))
 246                 .Where(uri=>uri.IsAbsoluteUri).ToArray();
 247
 248             SetSelectivePaths(selectiveUrls,null,null);
 249
 250             StartWatcherAgent();
 251
 252             StartNetworkAgent();
 253
 254             WorkflowAgent.RestartInterruptedFiles(_accountInfo);
 255             _started = true;
 256         }
 257
 258         private void EnsurePithosContainers()
 259         {
 260
 261             //Create the two default containers if they are missing
 262             var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
 263             foreach (var container in pithosContainers)
 264             {
 265                 var info=CloudClient.GetContainerInfo(UserName, container);
 266                 if (info == ContainerInfo.Empty)
 267                 {
 268                     CloudClient.CreateContainer(UserName, container);
 269                     info = CloudClient.GetContainerInfo(UserName, container);
 270                 }
 271                 _blockSize = info.BlockSize;
 272                 _blockHash = info.BlockHash;
 273                 _accountInfo.BlockSize = _blockSize;
 274                 _accountInfo.BlockHash = _blockHash;
 275             }
 276         }
 277
 278         public string AuthenticationUrl { get; set; }
 279
 280         private void IndexLocalFiles()
 281         {
 282             using (ThreadContext.Stacks["Operation"].Push("Indexing local files"))
 283             {
 284
 285                 try
 286                 {
 287                     //StatusNotification.NotifyChange("Indexing Local Files");
 288                     Log.Info("Start local indexing");
 289                     StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Indexing Local Files");
 290
 291                     var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
 292                     var directory = new DirectoryInfo(RootPath);
 293                     var files =
 294                         from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
 295                         where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
 296                               !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
 297                         select file;
 298                     StatusKeeper.ProcessExistingFiles(files);
 299
 300                 }
 301                 catch (Exception exc)
 302                 {
 303                     Log.Error("[ERROR]", exc);
 304                 }
 305                 finally
 306                 {
 307                     Log.Info("[END]");
 308                 }
 309                 StatusNotification.SetPithosStatus(PithosStatus.LocalComplete,"Indexing Completed");
 310             }
 311         }
 312
 313
 314
 315
 316
 317        /* private void StartWorkflowAgent()
 318         {
 319             WorkflowAgent.StatusNotification = StatusNotification;
 320
 321 /*            //On Vista and up we can check for a network connection
 322             bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
 323             //If we are not connected retry later
 324             if (!connected)
 325             {
 326                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
 327                 return;
 328             }#1#
 329
 330             try
 331             {
 332                 WorkflowAgent.Start();
 333             }
 334             catch (Exception)
 335             {
 336                 //Faild to authenticate due to network or account error
 337                 //Retry after a while
 338                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
 339             }
 340         }*/
 341
 342
 343         private void StartNetworkAgent()
 344         {
 345             NetworkAgent.StatusNotification = StatusNotification;
 346
 347             //TODO: The Network and Poll agents are not account specific
 348             //They should be moved outside PithosMonitor
 349             NetworkAgent.Start();
 350
 351             PollAgent.AddAccount(_accountInfo);
 352
 353             PollAgent.StatusNotification = StatusNotification;
 354
 355             PollAgent.PollRemoteFiles();
 356         }
 357
 358         //Make sure a hidden cache folder exists to store partial downloads
 359         private static void CreateHiddenFolder(string rootPath, string folderName)
 360         {
 361             if (String.IsNullOrWhiteSpace(rootPath))
 362                 throw new ArgumentNullException("rootPath");
 363             if (!Path.IsPathRooted(rootPath))
 364                 throw new ArgumentException("rootPath");
 365             if (String.IsNullOrWhiteSpace(folderName))
 366                 throw new ArgumentNullException("folderName");
 367             Contract.EndContractBlock();
 368
 369             var folder = Path.Combine(rootPath, folderName);
 370             if (!Directory.Exists(folder))
 371             {
 372                 var info = Directory.CreateDirectory(folder);
 373                 info.Attributes |= FileAttributes.Hidden;
 374
 375                 Log.InfoFormat("Created cache Folder: {0}", folder);
 376             }
 377             else
 378             {
 379                 var info = new DirectoryInfo(folder);
 380                 if ((info.Attributes & FileAttributes.Hidden) == 0)
 381                 {
 382                     info.Attributes |= FileAttributes.Hidden;
 383                     Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
 384                 }
 385             }
 386         }
 387
 388
 389
 390
 391         private void StartWatcherAgent()
 392         {
 393             if (Log.IsDebugEnabled)
 394                 Log.DebugFormat("Start Folder Monitoring [{0}]",RootPath);
 395
 396             AgentLocator<FileAgent>.Register(FileAgent,RootPath);
 397
 398             FileAgent.IdleTimeout = Settings.FileIdleTimeout;
 399             FileAgent.StatusKeeper = StatusKeeper;
 400             FileAgent.StatusNotification = StatusNotification;
 401             FileAgent.Workflow = Workflow;
 402             FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
 403             FileAgent.Start(_accountInfo, RootPath);
 404         }
 405
 406         public void Stop()
 407         {
 408             AgentLocator<FileAgent>.Remove(RootPath);
 409
 410             if (FileAgent!=null)
 411                 FileAgent.Stop();
 412             FileAgent = null;
 413         }
 414
 415
 416         ~PithosMonitor()
 417         {
 418             Dispose(false);
 419         }
 420
 421         public void Dispose()
 422         {
 423             Dispose(true);
 424             GC.SuppressFinalize(this);
 425         }
 426
 427         protected virtual void Dispose(bool disposing)
 428         {
 429             if (disposing)
 430             {
 431                 Stop();
 432             }
 433         }
 434
 435
 436         public void MoveFileStates(string oldPath, string newPath)
 437         {
 438             if (String.IsNullOrWhiteSpace(oldPath))
 439                 throw new ArgumentNullException("oldPath");
 440             if (!Path.IsPathRooted(oldPath))
 441                 throw new ArgumentException("oldPath must be an absolute path","oldPath");
 442             if (string.IsNullOrWhiteSpace(newPath))
 443                 throw new ArgumentNullException("newPath");
 444             if (!Path.IsPathRooted(newPath))
 445                 throw new ArgumentException("newPath must be an absolute path","newPath");
 446             Contract.EndContractBlock();
 447
 448             StatusKeeper.ChangeRoots(oldPath, newPath);
 449         }
 450
 451         public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed)
 452         {
 453             //Convert the uris to paths
 454             var selectivePaths = UrisToFilePaths(uris);
 455
 456             var selectiveUri = uris.ToList();
 457             this.Selectives.SetSelectedUris(_accountInfo,selectiveUri);
 458
 459             var removedPaths = UrisToFilePaths(removed);
 460             UnversionSelectivePaths(removedPaths);
 461
 462         }
 463
 464         /// <summary>
 465         /// Mark all unselected paths as Unversioned
 466         /// </summary>
 467         /// <param name="removed"></param>
 468         private void UnversionSelectivePaths(List<string> removed)
 469         {
 470             if (removed == null)
 471                 return;
 472
 473             //Ensure we remove any file state below the deleted folders
 474             FileState.UnversionPaths(removed);
 475         }
 476
 477
 478         /// <summary>
 479         /// Return a list of absolute filepaths from a list of Uris
 480         /// </summary>
 481         /// <param name="uris"></param>
 482         /// <returns></returns>
 483         private List<string> UrisToFilePaths(IEnumerable<Uri> uris)
 484         {
 485             if (uris == null)
 486                 return new List<string>();
 487
 488             var own = (from uri in uris
 489                        where uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())
 490                                    let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()
 491                                    //Trim the account name
 492                                    select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList();
 493             var others= (from uri in uris
 494                          where !uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())
 495                                    let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()
 496                                    //Trim the account name
 497                                    select Path.Combine(RootPath,"others-shared", relativePath)).ToList();
 498             return own.Union(others).ToList();
 499         }
 500
 501
 502         public ObjectInfo GetObjectInfo(string filePath)
 503         {
 504             if (String.IsNullOrWhiteSpace(filePath))
 505                 throw new ArgumentNullException("filePath");
 506             Contract.EndContractBlock();
 507
 508             var file=new FileInfo(filePath);
 509             string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
 510             var relativePath = file.AsRelativeTo(RootPath);
 511
 512             string accountName,container;
 513
 514             var parts=relativePath.Split('\\');
 515
 516             var accountInfo = _accountInfo;
 517             if (relativePath.StartsWith(FolderConstants.OthersFolder))
 518             {
 519                 accountName = parts[1];
 520                 container = parts[2];
 521                 relativeUrl = String.Join("/", parts.Splice(3));
 522                 //Create the root URL for the target account
 523                 var oldName = UserName;
 524                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
 525                 var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
 526                 var root=absoluteUri.Substring(0, nameIndex);
 527
 528                 accountInfo = new AccountInfo
 529                 {
 530                     UserName = accountName,
 531                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 532                     StorageUri = new Uri(root + accountName),
 533                     BlockHash=accountInfo.BlockHash,
 534                     BlockSize=accountInfo.BlockSize,
 535                     Token=accountInfo.Token
 536                 };
 537             }
 538             else
 539             {
 540                 accountName = UserName;
 541                 container = parts[0];
 542                 relativeUrl = String.Join("/", parts.Splice(1));
 543             }
 544
 545             var client = new CloudFilesClient(accountInfo);
 546             var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
 547             return objectInfo;
 548         }
 549
 550         public Task<ContainerInfo> GetContainerInfo(string filePath)
 551         {
 552             if (String.IsNullOrWhiteSpace(filePath))
 553                 throw new ArgumentNullException("filePath");
 554             Contract.EndContractBlock();
 555
 556             var file=new FileInfo(filePath);
 557             var relativePath = file.AsRelativeTo(RootPath);
 558
 559             string accountName,container;
 560
 561             var parts=relativePath.Split('\\');
 562
 563             var accountInfo = _accountInfo;
 564             if (relativePath.StartsWith(FolderConstants.OthersFolder))
 565             {
 566                 accountName = parts[1];
 567                 container = parts[2];
 568                 //Create the root URL for the target account
 569                 var oldName = UserName;
 570                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
 571                 var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
 572                 var root=absoluteUri.Substring(0, nameIndex);
 573
 574                 accountInfo = new AccountInfo
 575                 {
 576                     UserName = accountName,
 577                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 578                     StorageUri = new Uri(root + accountName),
 579                     BlockHash=accountInfo.BlockHash,
 580                     BlockSize=accountInfo.BlockSize,
 581                     Token=accountInfo.Token
 582                 };
 583             }
 584             else
 585             {
 586                 accountName = UserName;
 587                 container = parts[0];
 588             }
 589
 590             return Task.Factory.StartNew(() =>
 591             {
 592                 var client = new CloudFilesClient(accountInfo);
 593                 var containerInfo = client.GetContainerInfo(accountName, container);
 594                 return containerInfo;
 595             });
 596         }
 597     }
 598 }