code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/PithosMonitor.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="PithosMonitor.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System;
  43 using System.Collections.Generic;
  44 using System.ComponentModel.Composition;
  45 using System.Diagnostics.Contracts;
  46 using System.IO;
  47 using System.Linq;
  48 using System.Reflection;
  49 using System.Threading;
  50 using System.Threading.Tasks;
  51 using Pithos.Core.Agents;
  52 using Pithos.Interfaces;
  53 using Pithos.Network;
  54 using log4net;
  55
  56 namespace Pithos.Core
  57 {
  58     [Export(typeof(PithosMonitor))]
  59     public class PithosMonitor:IDisposable
  60     {
  61         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  62
  63         private int _blockSize;
  64         private string _blockHash;
  65
  66         [Import]
  67         public IPithosSettings Settings{get;set;}
  68
  69         private IStatusKeeper _statusKeeper;
  70
  71         [Import]
  72         public IStatusKeeper StatusKeeper
  73         {
  74             get { return _statusKeeper; }
  75             set
  76             {
  77                 _statusKeeper = value;
  78                 FileAgent.StatusKeeper = value;
  79             }
  80         }
  81
  82
  83         private IPithosWorkflow _workflow;
  84
  85         [Import]
  86         public IPithosWorkflow Workflow
  87         {
  88             get { return _workflow; }
  89             set
  90             {
  91                 _workflow = value;
  92                 FileAgent.Workflow = value;
  93             }
  94         }
  95
  96         public ICloudClient CloudClient { get; set; }
  97
  98         public IStatusNotification StatusNotification { get; set; }
  99
 100         //[Import]
 101         public FileAgent FileAgent { get; private set; }
 102
 103         private WorkflowAgent _workflowAgent;
 104
 105         [Import]
 106         public WorkflowAgent WorkflowAgent
 107         {
 108             get { return _workflowAgent; }
 109             set
 110             {
 111                 _workflowAgent = value;
 112                 FileAgent.WorkflowAgent = value;
 113             }
 114         }
 115
 116         [Import]
 117         public NetworkAgent NetworkAgent { get; set; }
 118         [Import]
 119         public PollAgent PollAgent { get; set; }
 120
 121         public string UserName { get; set; }
 122         private string _apiKey;
 123         public string ApiKey
 124         {
 125             get { return _apiKey; }
 126             set
 127             {
 128                 _apiKey = value;
 129                 if (_accountInfo != null)
 130                     _accountInfo.Token = value;
 131             }
 132         }
 133
 134         private AccountInfo _accountInfo;
 135
 136
 137
 138
 139
 140
 141         public bool Pause
 142         {
 143             get { return FileAgent.Pause; }
 144             set
 145             {
 146                 FileAgent.Pause = value;
 147             }
 148         }
 149
 150         private string _rootPath;
 151         public string RootPath
 152         {
 153             get { return _rootPath; }
 154             set
 155             {
 156                 _rootPath = String.IsNullOrWhiteSpace(value)
 157                     ? String.Empty
 158                     : value.ToLower();
 159             }
 160         }
 161
 162
 163         CancellationTokenSource _cancellationSource;
 164
 165         public PithosMonitor()
 166         {
 167             FileAgent = new FileAgent();
 168
 169         }
 170         private bool _started;
 171
 172         public void Start()
 173         {
 174             if (String.IsNullOrWhiteSpace(ApiKey))
 175                 throw new InvalidOperationException("The ApiKey is empty");
 176             if (String.IsNullOrWhiteSpace(UserName))
 177                 throw new InvalidOperationException("The UserName is empty");
 178             if (String.IsNullOrWhiteSpace(AuthenticationUrl))
 179                 throw new InvalidOperationException("The Authentication url is empty");
 180             Contract.EndContractBlock();
 181
 182             //If the account doesn't have a valid path, don't start monitoring but don't throw either
 183             if (String.IsNullOrWhiteSpace(RootPath))
 184                 //TODO; Warn user?
 185                 return;
 186
 187             WorkflowAgent.StatusNotification = StatusNotification;
 188
 189             StatusNotification.NotifyChange("Starting");
 190             if (_started)
 191             {
 192                 if (!_cancellationSource.IsCancellationRequested)
 193                     return;
 194             }
 195             _cancellationSource = new CancellationTokenSource();
 196
 197
 198             CloudClient = new CloudFilesClient(UserName, ApiKey)
 199                               {UsePithos = true, AuthenticationUrl = AuthenticationUrl};
 200
 201
 202             _accountInfo = CloudClient.Authenticate();
 203             _accountInfo.SiteUri = AuthenticationUrl;
 204             _accountInfo.AccountPath = RootPath;
 205
 206
 207             var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
 208             if (!Directory.Exists(pithosFolder))
 209                 Directory.CreateDirectory(pithosFolder);
 210             //Create the cache folder and ensure it is hidden
 211             CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
 212
 213             var policy=CloudClient.GetAccountPolicies(_accountInfo);
 214
 215             StatusNotification.NotifyAccount(policy);
 216             EnsurePithosContainers();
 217
 218             StatusKeeper.BlockHash = _blockHash;
 219             StatusKeeper.BlockSize = _blockSize;
 220
 221             StatusKeeper.StartProcessing(_cancellationSource.Token);
 222             IndexLocalFiles();
 223             //Extract the URIs from the string collection
 224             var settings = Settings.Accounts.First(s => s.AccountName == _accountInfo.UserName);
 225             var selectiveUrls=settings.SelectiveFolders.Cast<string>().Select(url => new Uri(url)).ToArray();
 226
 227             SetSelectivePaths(selectiveUrls,null,null);
 228
 229             StartWatcherAgent();
 230
 231             StartNetworkAgent();
 232
 233             WorkflowAgent.RestartInterruptedFiles(_accountInfo);
 234             _started = true;
 235         }
 236
 237         private void EnsurePithosContainers()
 238         {
 239
 240             //Create the two default containers if they are missing
 241             var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
 242             foreach (var container in pithosContainers)
 243             {
 244                 var info=CloudClient.GetContainerInfo(UserName, container);
 245                 if (info == ContainerInfo.Empty)
 246                 {
 247                     CloudClient.CreateContainer(UserName, container);
 248                     info = CloudClient.GetContainerInfo(UserName, container);
 249                 }
 250                 _blockSize = info.BlockSize;
 251                 _blockHash = info.BlockHash;
 252                 _accountInfo.BlockSize = _blockSize;
 253                 _accountInfo.BlockHash = _blockHash;
 254             }
 255         }
 256
 257         public string AuthenticationUrl { get; set; }
 258
 259         private void IndexLocalFiles()
 260         {
 261             using (ThreadContext.Stacks["Operation"].Push("Indexing local files"))
 262             {
 263
 264                 try
 265                 {
 266                     //StatusNotification.NotifyChange("Indexing Local Files");
 267                     Log.Info("Start local indexing");
 268                     StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Indexing Local Files");
 269
 270                     var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
 271                     var directory = new DirectoryInfo(RootPath);
 272                     var files =
 273                         from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
 274                         where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
 275                               !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
 276                         select file;
 277                     StatusKeeper.ProcessExistingFiles(files);
 278
 279                 }
 280                 catch (Exception exc)
 281                 {
 282                     Log.Error("[ERROR]", exc);
 283                 }
 284                 finally
 285                 {
 286                     Log.Info("[END]");
 287                 }
 288                 StatusNotification.SetPithosStatus(PithosStatus.LocalComplete,"Indexing Completed");
 289             }
 290         }
 291
 292
 293
 294
 295
 296        /* private void StartWorkflowAgent()
 297         {
 298             WorkflowAgent.StatusNotification = StatusNotification;
 299
 300 /*            //On Vista and up we can check for a network connection
 301             bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
 302             //If we are not connected retry later
 303             if (!connected)
 304             {
 305                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
 306                 return;
 307             }#1#
 308
 309             try
 310             {
 311                 WorkflowAgent.Start();
 312             }
 313             catch (Exception)
 314             {
 315                 //Faild to authenticate due to network or account error
 316                 //Retry after a while
 317                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
 318             }
 319         }*/
 320
 321         internal class LocalFileComparer:EqualityComparer<CloudAction>
 322         {
 323             public override bool Equals(CloudAction x, CloudAction y)
 324             {
 325                 if (x.Action != y.Action)
 326                     return false;
 327                 if (x.LocalFile != null && y.LocalFile != null && !x.LocalFile.FullName.Equals(y.LocalFile.FullName))
 328                     return false;
 329                 if (x.CloudFile != null && y.CloudFile != null )
 330                 {
 331                     if (x.CloudFile.Hash == null & y.CloudFile.Hash != null)
 332                         return false;
 333                     if (x.CloudFile.Hash != null & y.CloudFile.Hash == null)
 334                         return false;
 335                     if (x.CloudFile.Hash == null & y.CloudFile.Hash == null)
 336                         return (x.CloudFile.Name == y.CloudFile.Name);
 337                     if (!x.CloudFile.Hash.Equals(y.CloudFile.Hash))
 338                         return false;
 339                 }
 340                 if (x.CloudFile == null ^ y.CloudFile == null ||
 341                     x.LocalFile == null ^ y.LocalFile == null)
 342                     return false;
 343                 return true;
 344             }
 345
 346             public override int GetHashCode(CloudAction obj)
 347             {
 348                 if (obj == null)
 349                     return 0;
 350                 var hash1 = (obj.LocalFile == null) ? int.MaxValue : obj.LocalFile.FullName.GetHashCode();
 351                 var hash2 = (obj.CloudFile == null) ? int.MaxValue : (obj.CloudFile.Hash ?? obj.CloudFile.Name??"").GetHashCode();
 352                 var hash3 = obj.Action.GetHashCode();
 353                 return hash1 ^ hash2 & hash3;
 354             }
 355         }
 356
 357
 358         private void StartNetworkAgent()
 359         {
 360             NetworkAgent.StatusNotification = StatusNotification;
 361
 362             //TODO: The Network and Poll agents are not account specific
 363             //They should be moved outside PithosMonitor
 364             NetworkAgent.Start();
 365
 366             PollAgent.AddAccount(_accountInfo);
 367
 368             PollAgent.StatusNotification = StatusNotification;
 369
 370             PollAgent.PollRemoteFiles();
 371         }
 372
 373         //Make sure a hidden cache folder exists to store partial downloads
 374         private static void CreateHiddenFolder(string rootPath, string folderName)
 375         {
 376             if (String.IsNullOrWhiteSpace(rootPath))
 377                 throw new ArgumentNullException("rootPath");
 378             if (!Path.IsPathRooted(rootPath))
 379                 throw new ArgumentException("rootPath");
 380             if (String.IsNullOrWhiteSpace(folderName))
 381                 throw new ArgumentNullException("folderName");
 382             Contract.EndContractBlock();
 383
 384             var folder = Path.Combine(rootPath, folderName);
 385             if (!Directory.Exists(folder))
 386             {
 387                 var info = Directory.CreateDirectory(folder);
 388                 info.Attributes |= FileAttributes.Hidden;
 389
 390                 Log.InfoFormat("Created cache Folder: {0}", folder);
 391             }
 392             else
 393             {
 394                 var info = new DirectoryInfo(folder);
 395                 if ((info.Attributes & FileAttributes.Hidden) == 0)
 396                 {
 397                     info.Attributes |= FileAttributes.Hidden;
 398                     Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
 399                 }
 400             }
 401         }
 402
 403
 404
 405
 406         private void StartWatcherAgent()
 407         {
 408             if (Log.IsDebugEnabled)
 409                 Log.DebugFormat("Start Folder Monitoring [{0}]",RootPath);
 410
 411             AgentLocator<FileAgent>.Register(FileAgent,RootPath);
 412
 413             FileAgent.StatusKeeper = StatusKeeper;
 414             FileAgent.StatusNotification = StatusNotification;
 415             FileAgent.Workflow = Workflow;
 416             FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
 417             FileAgent.Start(_accountInfo, RootPath);
 418         }
 419
 420         public void Stop()
 421         {
 422             AgentLocator<FileAgent>.Remove(RootPath);
 423
 424             if (FileAgent!=null)
 425                 FileAgent.Stop();
 426             FileAgent = null;
 427         }
 428
 429
 430         ~PithosMonitor()
 431         {
 432             Dispose(false);
 433         }
 434
 435         public void Dispose()
 436         {
 437             Dispose(true);
 438             GC.SuppressFinalize(this);
 439         }
 440
 441         protected virtual void Dispose(bool disposing)
 442         {
 443             if (disposing)
 444             {
 445                 Stop();
 446             }
 447         }
 448
 449
 450         public void MoveFileStates(string oldPath, string newPath)
 451         {
 452             if (String.IsNullOrWhiteSpace(oldPath))
 453                 throw new ArgumentNullException("oldPath");
 454             if (!Path.IsPathRooted(oldPath))
 455                 throw new ArgumentException("oldPath must be an absolute path","oldPath");
 456             if (string.IsNullOrWhiteSpace(newPath))
 457                 throw new ArgumentNullException("newPath");
 458             if (!Path.IsPathRooted(newPath))
 459                 throw new ArgumentException("newPath must be an absolute path","newPath");
 460             Contract.EndContractBlock();
 461
 462             StatusKeeper.ChangeRoots(oldPath, newPath);
 463         }
 464
 465         public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed)
 466         {
 467             //Convert the uris to paths
 468             var selectivePaths = UrisToFilePaths(uris);
 469
 470             FileAgent.SelectivePaths=selectivePaths;
 471             PollAgent.SetSyncUris(uris);
 472
 473             var removedPaths = UrisToFilePaths(removed);
 474             UnversionSelectivePaths(removedPaths);
 475
 476         }
 477
 478         /// <summary>
 479         /// Mark all unselected paths as Unversioned
 480         /// </summary>
 481         /// <param name="removed"></param>
 482         private void UnversionSelectivePaths(List<string> removed)
 483         {
 484             if (removed == null)
 485                 return;
 486
 487             //Ensure we remove any file state below the deleted folders
 488             FileState.UnversionPaths(removed);
 489         }
 490
 491
 492         /// <summary>
 493         /// Return a list of absolute filepaths from a list of Uris
 494         /// </summary>
 495         /// <param name="uris"></param>
 496         /// <returns></returns>
 497         private List<string> UrisToFilePaths(IEnumerable<Uri> uris)
 498         {
 499             if (uris == null)
 500                 return new List<string>();
 501
 502             return (from uri in uris
 503                     let relativePath = _accountInfo.StorageUri
 504                         .MakeRelativeUri(uri)
 505                         .RelativeUriToFilePath()
 506                         //Trim the account name
 507                     select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList();
 508         }
 509
 510
 511         public ObjectInfo GetObjectInfo(string filePath)
 512         {
 513             if (String.IsNullOrWhiteSpace(filePath))
 514                 throw new ArgumentNullException("filePath");
 515             Contract.EndContractBlock();
 516
 517             var file=new FileInfo(filePath);
 518             string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
 519             var relativePath = file.AsRelativeTo(RootPath);
 520
 521             string accountName,container;
 522
 523             var parts=relativePath.Split('\\');
 524
 525             var accountInfo = _accountInfo;
 526             if (relativePath.StartsWith(FolderConstants.OthersFolder))
 527             {
 528                 accountName = parts[1];
 529                 container = parts[2];
 530                 relativeUrl = String.Join("/", parts.Splice(3));
 531                 //Create the root URL for the target account
 532                 var oldName = UserName;
 533                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
 534                 var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
 535                 var root=absoluteUri.Substring(0, nameIndex);
 536
 537                 accountInfo = new AccountInfo
 538                 {
 539                     UserName = accountName,
 540                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 541                     StorageUri = new Uri(root + accountName),
 542                     BlockHash=accountInfo.BlockHash,
 543                     BlockSize=accountInfo.BlockSize,
 544                     Token=accountInfo.Token
 545                 };
 546             }
 547             else
 548             {
 549                 accountName = UserName;
 550                 container = parts[0];
 551                 relativeUrl = String.Join("/", parts.Splice(1));
 552             }
 553
 554             var client = new CloudFilesClient(accountInfo);
 555             var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
 556             return objectInfo;
 557         }
 558
 559         public Task<ContainerInfo> GetContainerInfo(string filePath)
 560         {
 561             if (String.IsNullOrWhiteSpace(filePath))
 562                 throw new ArgumentNullException("filePath");
 563             Contract.EndContractBlock();
 564
 565             var file=new FileInfo(filePath);
 566             var relativePath = file.AsRelativeTo(RootPath);
 567
 568             string accountName,container;
 569
 570             var parts=relativePath.Split('\\');
 571
 572             var accountInfo = _accountInfo;
 573             if (relativePath.StartsWith(FolderConstants.OthersFolder))
 574             {
 575                 accountName = parts[1];
 576                 container = parts[2];
 577                 //Create the root URL for the target account
 578                 var oldName = UserName;
 579                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
 580                 var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
 581                 var root=absoluteUri.Substring(0, nameIndex);
 582
 583                 accountInfo = new AccountInfo
 584                 {
 585                     UserName = accountName,
 586                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 587                     StorageUri = new Uri(root + accountName),
 588                     BlockHash=accountInfo.BlockHash,
 589                     BlockSize=accountInfo.BlockSize,
 590                     Token=accountInfo.Token
 591                 };
 592             }
 593             else
 594             {
 595                 accountName = UserName;
 596                 container = parts[0];
 597             }
 598
 599             return Task.Factory.StartNew(() =>
 600             {
 601                 var client = new CloudFilesClient(accountInfo);
 602                 var containerInfo = client.GetContainerInfo(accountName, container);
 603                 return containerInfo;
 604             });
 605         }
 606     }
 607 }