code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/PithosMonitor.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="PithosMonitor.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System;
  43 using System.Collections.Generic;
  44 using System.ComponentModel.Composition;
  45 using System.Diagnostics.Contracts;
  46 using System.IO;
  47 using System.Linq;
  48 using System.Reflection;
  49 using System.Threading;
  50 using System.Threading.Tasks;
  51 using Pithos.Core.Agents;
  52 using Pithos.Interfaces;
  53 using Pithos.Network;
  54 using log4net;
  55
  56 namespace Pithos.Core
  57 {
  58     [Export(typeof(PithosMonitor))]
  59     public class PithosMonitor:IDisposable
  60     {
  61         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  62
  63         private int _blockSize;
  64         private string _blockHash;
  65
  66         [Import]
  67         public IPithosSettings Settings{get;set;}
  68
  69         private IStatusKeeper _statusKeeper;
  70
  71         [Import]
  72         public IStatusKeeper StatusKeeper
  73         {
  74             get { return _statusKeeper; }
  75             set
  76             {
  77                 _statusKeeper = value;
  78                 FileAgent.StatusKeeper = value;
  79             }
  80         }
  81
  82
  83         private IPithosWorkflow _workflow;
  84
  85         [Import]
  86         public IPithosWorkflow Workflow
  87         {
  88             get { return _workflow; }
  89             set
  90             {
  91                 _workflow = value;
  92                 FileAgent.Workflow = value;
  93             }
  94         }
  95
  96         public ICloudClient CloudClient { get; set; }
  97
  98         public IStatusNotification StatusNotification { get; set; }
  99
 100         //[Import]
 101         public FileAgent FileAgent { get; private set; }
 102
 103         private WorkflowAgent _workflowAgent;
 104
 105         [Import]
 106         public WorkflowAgent WorkflowAgent
 107         {
 108             get { return _workflowAgent; }
 109             set
 110             {
 111                 _workflowAgent = value;
 112                 FileAgent.WorkflowAgent = value;
 113             }
 114         }
 115
 116         [Import]
 117         public NetworkAgent NetworkAgent { get; set; }
 118         [Import]
 119         public PollAgent PollAgent { get; set; }
 120
 121         public string UserName { get; set; }
 122         private string _apiKey;
 123         public string ApiKey
 124         {
 125             get { return _apiKey; }
 126             set
 127             {
 128                 _apiKey = value;
 129                 if (_accountInfo != null)
 130                     _accountInfo.Token = value;
 131             }
 132         }
 133
 134         private AccountInfo _accountInfo;
 135
 136
 137
 138
 139
 140
 141         public bool Pause
 142         {
 143             get { return FileAgent.Pause; }
 144             set
 145             {
 146                 FileAgent.Pause = value;
 147             }
 148         }
 149
 150         private string _rootPath;
 151         public string RootPath
 152         {
 153             get { return _rootPath; }
 154             set
 155             {
 156                 _rootPath = String.IsNullOrWhiteSpace(value)
 157                     ? String.Empty
 158                     : value.ToLower();
 159             }
 160         }
 161
 162
 163         CancellationTokenSource _cancellationSource;
 164
 165         public PithosMonitor()
 166         {
 167             FileAgent = new FileAgent();
 168         }
 169         private bool _started;
 170
 171         public void Start()
 172         {
 173             if (String.IsNullOrWhiteSpace(ApiKey))
 174                 throw new InvalidOperationException("The ApiKey is empty");
 175             if (String.IsNullOrWhiteSpace(UserName))
 176                 throw new InvalidOperationException("The UserName is empty");
 177             if (String.IsNullOrWhiteSpace(AuthenticationUrl))
 178                 throw new InvalidOperationException("The Authentication url is empty");
 179             Contract.EndContractBlock();
 180
 181             //If the account doesn't have a valid path, don't start monitoring but don't throw either
 182             if (String.IsNullOrWhiteSpace(RootPath))
 183                 //TODO; Warn user?
 184                 return;
 185
 186             WorkflowAgent.StatusNotification = StatusNotification;
 187
 188             StatusNotification.NotifyChange("Starting");
 189             if (_started)
 190             {
 191                 if (!_cancellationSource.IsCancellationRequested)
 192                     return;
 193             }
 194             _cancellationSource = new CancellationTokenSource();
 195
 196             lock (this)
 197             {
 198                 CloudClient = new CloudFilesClient(UserName, ApiKey)
 199                                   {UsePithos = true, AuthenticationUrl = AuthenticationUrl};
 200                 _accountInfo = CloudClient.Authenticate();
 201             }
 202             _accountInfo.SiteUri = AuthenticationUrl;
 203             _accountInfo.AccountPath = RootPath;
 204
 205
 206             var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
 207             if (!Directory.Exists(pithosFolder))
 208                 Directory.CreateDirectory(pithosFolder);
 209             //Create the cache folder and ensure it is hidden
 210             CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
 211
 212             var policy=CloudClient.GetAccountPolicies(_accountInfo);
 213
 214             StatusNotification.NotifyAccount(policy);
 215             EnsurePithosContainers();
 216
 217             StatusKeeper.BlockHash = _blockHash;
 218             StatusKeeper.BlockSize = _blockSize;
 219
 220             StatusKeeper.StartProcessing(_cancellationSource.Token);
 221             IndexLocalFiles();
 222             //Extract the URIs from the string collection
 223             var settings = Settings.Accounts.First(s => s.AccountKey == _accountInfo.AccountKey );
 224             var selectiveUrls=settings.SelectiveFolders.Cast<string>().Select(url => new Uri(url)).ToArray();
 225
 226             SetSelectivePaths(selectiveUrls,null,null);
 227
 228             StartWatcherAgent();
 229
 230             StartNetworkAgent();
 231
 232             WorkflowAgent.RestartInterruptedFiles(_accountInfo);
 233             _started = true;
 234         }
 235
 236         private void EnsurePithosContainers()
 237         {
 238
 239             //Create the two default containers if they are missing
 240             var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
 241             foreach (var container in pithosContainers)
 242             {
 243                 var info=CloudClient.GetContainerInfo(UserName, container);
 244                 if (info == ContainerInfo.Empty)
 245                 {
 246                     CloudClient.CreateContainer(UserName, container);
 247                     info = CloudClient.GetContainerInfo(UserName, container);
 248                 }
 249                 _blockSize = info.BlockSize;
 250                 _blockHash = info.BlockHash;
 251                 _accountInfo.BlockSize = _blockSize;
 252                 _accountInfo.BlockHash = _blockHash;
 253             }
 254         }
 255
 256         public string AuthenticationUrl { get; set; }
 257
 258         private void IndexLocalFiles()
 259         {
 260             using (ThreadContext.Stacks["Operation"].Push("Indexing local files"))
 261             {
 262
 263                 try
 264                 {
 265                     //StatusNotification.NotifyChange("Indexing Local Files");
 266                     Log.Info("Start local indexing");
 267                     StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Indexing Local Files");
 268
 269                     var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
 270                     var directory = new DirectoryInfo(RootPath);
 271                     var files =
 272                         from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
 273                         where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
 274                               !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
 275                         select file;
 276                     StatusKeeper.ProcessExistingFiles(files);
 277
 278                 }
 279                 catch (Exception exc)
 280                 {
 281                     Log.Error("[ERROR]", exc);
 282                 }
 283                 finally
 284                 {
 285                     Log.Info("[END]");
 286                 }
 287                 StatusNotification.SetPithosStatus(PithosStatus.LocalComplete,"Indexing Completed");
 288             }
 289         }
 290
 291
 292
 293
 294
 295        /* private void StartWorkflowAgent()
 296         {
 297             WorkflowAgent.StatusNotification = StatusNotification;
 298
 299 /*            //On Vista and up we can check for a network connection
 300             bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
 301             //If we are not connected retry later
 302             if (!connected)
 303             {
 304                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
 305                 return;
 306             }#1#
 307
 308             try
 309             {
 310                 WorkflowAgent.Start();
 311             }
 312             catch (Exception)
 313             {
 314                 //Faild to authenticate due to network or account error
 315                 //Retry after a while
 316                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
 317             }
 318         }*/
 319
 320
 321         private void StartNetworkAgent()
 322         {
 323             NetworkAgent.StatusNotification = StatusNotification;
 324
 325             //TODO: The Network and Poll agents are not account specific
 326             //They should be moved outside PithosMonitor
 327             NetworkAgent.Start();
 328
 329             PollAgent.AddAccount(_accountInfo);
 330
 331             PollAgent.StatusNotification = StatusNotification;
 332
 333             PollAgent.PollRemoteFiles();
 334         }
 335
 336         //Make sure a hidden cache folder exists to store partial downloads
 337         private static void CreateHiddenFolder(string rootPath, string folderName)
 338         {
 339             if (String.IsNullOrWhiteSpace(rootPath))
 340                 throw new ArgumentNullException("rootPath");
 341             if (!Path.IsPathRooted(rootPath))
 342                 throw new ArgumentException("rootPath");
 343             if (String.IsNullOrWhiteSpace(folderName))
 344                 throw new ArgumentNullException("folderName");
 345             Contract.EndContractBlock();
 346
 347             var folder = Path.Combine(rootPath, folderName);
 348             if (!Directory.Exists(folder))
 349             {
 350                 var info = Directory.CreateDirectory(folder);
 351                 info.Attributes |= FileAttributes.Hidden;
 352
 353                 Log.InfoFormat("Created cache Folder: {0}", folder);
 354             }
 355             else
 356             {
 357                 var info = new DirectoryInfo(folder);
 358                 if ((info.Attributes & FileAttributes.Hidden) == 0)
 359                 {
 360                     info.Attributes |= FileAttributes.Hidden;
 361                     Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
 362                 }
 363             }
 364         }
 365
 366
 367
 368
 369         private void StartWatcherAgent()
 370         {
 371             if (Log.IsDebugEnabled)
 372                 Log.DebugFormat("Start Folder Monitoring [{0}]",RootPath);
 373
 374             AgentLocator<FileAgent>.Register(FileAgent,RootPath);
 375
 376             FileAgent.IdleTimeout = Settings.FileIdleTimeout;
 377             FileAgent.StatusKeeper = StatusKeeper;
 378             FileAgent.StatusNotification = StatusNotification;
 379             FileAgent.Workflow = Workflow;
 380             FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
 381             FileAgent.Start(_accountInfo, RootPath);
 382         }
 383
 384         public void Stop()
 385         {
 386             AgentLocator<FileAgent>.Remove(RootPath);
 387
 388             if (FileAgent!=null)
 389                 FileAgent.Stop();
 390             FileAgent = null;
 391         }
 392
 393
 394         ~PithosMonitor()
 395         {
 396             Dispose(false);
 397         }
 398
 399         public void Dispose()
 400         {
 401             Dispose(true);
 402             GC.SuppressFinalize(this);
 403         }
 404
 405         protected virtual void Dispose(bool disposing)
 406         {
 407             if (disposing)
 408             {
 409                 Stop();
 410             }
 411         }
 412
 413
 414         public void MoveFileStates(string oldPath, string newPath)
 415         {
 416             if (String.IsNullOrWhiteSpace(oldPath))
 417                 throw new ArgumentNullException("oldPath");
 418             if (!Path.IsPathRooted(oldPath))
 419                 throw new ArgumentException("oldPath must be an absolute path","oldPath");
 420             if (string.IsNullOrWhiteSpace(newPath))
 421                 throw new ArgumentNullException("newPath");
 422             if (!Path.IsPathRooted(newPath))
 423                 throw new ArgumentException("newPath must be an absolute path","newPath");
 424             Contract.EndContractBlock();
 425
 426             StatusKeeper.ChangeRoots(oldPath, newPath);
 427         }
 428
 429         public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed)
 430         {
 431             //Convert the uris to paths
 432             var selectivePaths = UrisToFilePaths(uris);
 433
 434             FileAgent.SelectivePaths=selectivePaths;
 435             WorkflowAgent.SelectivePaths = selectivePaths;
 436             PollAgent.SetSyncUris(_accountInfo.AccountKey,uris);
 437
 438             var removedPaths = UrisToFilePaths(removed);
 439             UnversionSelectivePaths(removedPaths);
 440
 441         }
 442
 443         /// <summary>
 444         /// Mark all unselected paths as Unversioned
 445         /// </summary>
 446         /// <param name="removed"></param>
 447         private void UnversionSelectivePaths(List<string> removed)
 448         {
 449             if (removed == null)
 450                 return;
 451
 452             //Ensure we remove any file state below the deleted folders
 453             FileState.UnversionPaths(removed);
 454         }
 455
 456
 457         /// <summary>
 458         /// Return a list of absolute filepaths from a list of Uris
 459         /// </summary>
 460         /// <param name="uris"></param>
 461         /// <returns></returns>
 462         private List<string> UrisToFilePaths(IEnumerable<Uri> uris)
 463         {
 464             if (uris == null)
 465                 return new List<string>();
 466
 467             var own = (from uri in uris
 468                        where uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())
 469                                    let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()
 470                                    //Trim the account name
 471                                    select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList();
 472             var others= (from uri in uris
 473                          where !uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())
 474                                    let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()
 475                                    //Trim the account name
 476                                    select Path.Combine(RootPath,"others-shared", relativePath)).ToList();
 477             return own.Union(others).ToList();
 478         }
 479
 480
 481         public ObjectInfo GetObjectInfo(string filePath)
 482         {
 483             if (String.IsNullOrWhiteSpace(filePath))
 484                 throw new ArgumentNullException("filePath");
 485             Contract.EndContractBlock();
 486
 487             var file=new FileInfo(filePath);
 488             string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
 489             var relativePath = file.AsRelativeTo(RootPath);
 490
 491             string accountName,container;
 492
 493             var parts=relativePath.Split('\\');
 494
 495             var accountInfo = _accountInfo;
 496             if (relativePath.StartsWith(FolderConstants.OthersFolder))
 497             {
 498                 accountName = parts[1];
 499                 container = parts[2];
 500                 relativeUrl = String.Join("/", parts.Splice(3));
 501                 //Create the root URL for the target account
 502                 var oldName = UserName;
 503                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
 504                 var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
 505                 var root=absoluteUri.Substring(0, nameIndex);
 506
 507                 accountInfo = new AccountInfo
 508                 {
 509                     UserName = accountName,
 510                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 511                     StorageUri = new Uri(root + accountName),
 512                     BlockHash=accountInfo.BlockHash,
 513                     BlockSize=accountInfo.BlockSize,
 514                     Token=accountInfo.Token
 515                 };
 516             }
 517             else
 518             {
 519                 accountName = UserName;
 520                 container = parts[0];
 521                 relativeUrl = String.Join("/", parts.Splice(1));
 522             }
 523
 524             var client = new CloudFilesClient(accountInfo);
 525             var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
 526             return objectInfo;
 527         }
 528
 529         public Task<ContainerInfo> GetContainerInfo(string filePath)
 530         {
 531             if (String.IsNullOrWhiteSpace(filePath))
 532                 throw new ArgumentNullException("filePath");
 533             Contract.EndContractBlock();
 534
 535             var file=new FileInfo(filePath);
 536             var relativePath = file.AsRelativeTo(RootPath);
 537
 538             string accountName,container;
 539
 540             var parts=relativePath.Split('\\');
 541
 542             var accountInfo = _accountInfo;
 543             if (relativePath.StartsWith(FolderConstants.OthersFolder))
 544             {
 545                 accountName = parts[1];
 546                 container = parts[2];
 547                 //Create the root URL for the target account
 548                 var oldName = UserName;
 549                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
 550                 var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
 551                 var root=absoluteUri.Substring(0, nameIndex);
 552
 553                 accountInfo = new AccountInfo
 554                 {
 555                     UserName = accountName,
 556                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 557                     StorageUri = new Uri(root + accountName),
 558                     BlockHash=accountInfo.BlockHash,
 559                     BlockSize=accountInfo.BlockSize,
 560                     Token=accountInfo.Token
 561                 };
 562             }
 563             else
 564             {
 565                 accountName = UserName;
 566                 container = parts[0];
 567             }
 568
 569             return Task.Factory.StartNew(() =>
 570             {
 571                 var client = new CloudFilesClient(accountInfo);
 572                 var containerInfo = client.GetContainerInfo(accountName, container);
 573                 return containerInfo;
 574             });
 575         }
 576     }
 577 }