code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/PithosMonitor.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="PithosMonitor.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System;
  43 using System.Collections.Concurrent;
  44 using System.Collections.Generic;
  45 using System.ComponentModel.Composition;
  46 using System.Diagnostics;
  47 using System.Diagnostics.Contracts;
  48 using System.IO;
  49 using System.Linq;
  50 using System.Net;
  51 using System.Net.NetworkInformation;
  52 using System.Security.Cryptography;
  53 using System.ServiceModel.Description;
  54 using System.Text;
  55 using System.Threading;
  56 using System.Threading.Tasks;
  57 using Castle.ActiveRecord.Queries;
  58 using Microsoft.WindowsAPICodePack.Net;
  59 using Pithos.Core.Agents;
  60 using Pithos.Interfaces;
  61 using System.ServiceModel;
  62 using Pithos.Network;
  63 using log4net;
  64
  65 namespace Pithos.Core
  66 {
  67     [Export(typeof(PithosMonitor))]
  68     public class PithosMonitor:IDisposable
  69     {
  70         private int _blockSize;
  71         private string _blockHash;
  72
  73         [Import]
  74         public IPithosSettings Settings{get;set;}
  75
  76         private IStatusKeeper _statusKeeper;
  77
  78         [Import]
  79         public IStatusKeeper StatusKeeper
  80         {
  81             get { return _statusKeeper; }
  82             set
  83             {
  84                 _statusKeeper = value;
  85                 FileAgent.StatusKeeper = value;
  86             }
  87         }
  88
  89
  90         private IPithosWorkflow _workflow;
  91
  92         [Import]
  93         public IPithosWorkflow Workflow
  94         {
  95             get { return _workflow; }
  96             set
  97             {
  98                 _workflow = value;
  99                 FileAgent.Workflow = value;
 100             }
 101         }
 102
 103         public ICloudClient CloudClient { get; set; }
 104
 105         public IStatusNotification StatusNotification { get; set; }
 106
 107         //[Import]
 108         public FileAgent FileAgent { get; private set; }
 109
 110         private WorkflowAgent _workflowAgent;
 111
 112         [Import]
 113         public WorkflowAgent WorkflowAgent
 114         {
 115             get { return _workflowAgent; }
 116             set
 117             {
 118                 _workflowAgent = value;
 119                 FileAgent.WorkflowAgent = value;
 120             }
 121         }
 122
 123         [Import]
 124         public NetworkAgent NetworkAgent { get; set; }
 125         [Import]
 126         public PollAgent PollAgent { get; set; }
 127
 128         public string UserName { get; set; }
 129         private string _apiKey;
 130         public string ApiKey
 131         {
 132             get { return _apiKey; }
 133             set
 134             {
 135                 _apiKey = value;
 136                 if (_accountInfo != null)
 137                     _accountInfo.Token = value;
 138             }
 139         }
 140
 141         private AccountInfo _accountInfo;
 142
 143
 144
 145
 146         private static readonly ILog Log = LogManager.GetLogger(typeof(PithosMonitor));
 147
 148
 149         public bool Pause
 150         {
 151             get { return FileAgent.Pause; }
 152             set
 153             {
 154                 FileAgent.Pause = value;
 155                 if (value)
 156                 {
 157                     StatusKeeper.SetPithosStatus(PithosStatus.SyncPaused);
 158                     StatusNotification.NotifyChange("Paused");
 159                 }
 160                 else
 161                 {
 162                     StatusKeeper.SetPithosStatus(PithosStatus.InSynch);
 163                     StatusNotification.NotifyChange("Synchronizing");
 164                 }
 165             }
 166         }
 167
 168         private string _rootPath;
 169         public string RootPath
 170         {
 171             get { return _rootPath; }
 172             set
 173             {
 174                 _rootPath = String.IsNullOrWhiteSpace(value)
 175                     ? String.Empty
 176                     : value.ToLower();
 177             }
 178         }
 179
 180
 181         CancellationTokenSource _cancellationSource;
 182
 183         public PithosMonitor()
 184         {
 185             FileAgent = new FileAgent();
 186
 187         }
 188         private bool _started;
 189
 190         public void Start()
 191         {
 192             if (String.IsNullOrWhiteSpace(ApiKey))
 193                 throw new InvalidOperationException("The ApiKey is empty");
 194             if (String.IsNullOrWhiteSpace(UserName))
 195                 throw new InvalidOperationException("The UserName is empty");
 196             if (String.IsNullOrWhiteSpace(AuthenticationUrl))
 197                 throw new InvalidOperationException("The Authentication url is empty");
 198             Contract.EndContractBlock();
 199
 200             //If the account doesn't have a valid path, don't start monitoring but don't throw either
 201             if (String.IsNullOrWhiteSpace(RootPath))
 202                 //TODO; Warn user?
 203                 return;
 204
 205             WorkflowAgent.StatusNotification = StatusNotification;
 206
 207             StatusNotification.NotifyChange("Starting");
 208             if (_started)
 209             {
 210                 if (!_cancellationSource.IsCancellationRequested)
 211                     return;
 212             }
 213             _cancellationSource = new CancellationTokenSource();
 214
 215
 216             CloudClient=new CloudFilesClient(UserName,ApiKey);
 217             CloudClient.UsePithos = true;
 218             CloudClient.AuthenticationUrl = this.AuthenticationUrl;
 219
 220             _accountInfo = CloudClient.Authenticate();
 221             _accountInfo.SiteUri = AuthenticationUrl;
 222             _accountInfo.AccountPath = RootPath;
 223
 224
 225             var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
 226             if (!Directory.Exists(pithosFolder))
 227                 Directory.CreateDirectory(pithosFolder);
 228             //Create the cache folder and ensure it is hidden
 229             CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
 230
 231             var policy=CloudClient.GetAccountPolicies(_accountInfo);
 232
 233             StatusNotification.NotifyAccount(policy);
 234             EnsurePithosContainers();
 235
 236             StatusKeeper.BlockHash = _blockHash;
 237             StatusKeeper.BlockSize = _blockSize;
 238
 239             StatusKeeper.StartProcessing(_cancellationSource.Token);
 240             IndexLocalFiles();
 241             StartWatcherAgent();
 242
 243             StartNetworkAgent();
 244
 245             WorkflowAgent.RestartInterruptedFiles(_accountInfo);
 246             _started = true;
 247         }
 248
 249         private void EnsurePithosContainers()
 250         {
 251
 252             //Create the two default containers if they are missing
 253             var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
 254             foreach (var container in pithosContainers)
 255             {
 256                 var info=CloudClient.GetContainerInfo(this.UserName, container);
 257                 if (info == ContainerInfo.Empty)
 258                 {
 259                     CloudClient.CreateContainer(this.UserName, container);
 260                     info = CloudClient.GetContainerInfo(this.UserName, container);
 261                 }
 262                 _blockSize = info.BlockSize;
 263                 _blockHash = info.BlockHash;
 264                 _accountInfo.BlockSize = _blockSize;
 265                 _accountInfo.BlockHash = _blockHash;
 266             }
 267         }
 268
 269         public string AuthenticationUrl { get; set; }
 270
 271         private void IndexLocalFiles()
 272         {
 273             StatusNotification.NotifyChange("Indexing Local Files",TraceLevel.Info);
 274             using (log4net.ThreadContext.Stacks["Monitor"].Push("Indexing local files"))
 275             {
 276                 Log.Info("START");
 277                 try
 278                 {
 279                     var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
 280                     var directory = new DirectoryInfo(RootPath);
 281                     var files =
 282                         from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
 283                         where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
 284                               !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
 285                         select file;
 286                     StatusKeeper.ProcessExistingFiles(files);
 287
 288                 }
 289                 catch (Exception exc)
 290                 {
 291                     Log.Error("[ERROR]", exc);
 292                 }
 293                 finally
 294                 {
 295                     Log.Info("[END]");
 296                 }
 297             }
 298         }
 299
 300
 301
 302
 303
 304        /* private void StartWorkflowAgent()
 305         {
 306             WorkflowAgent.StatusNotification = StatusNotification;
 307
 308 /*            //On Vista and up we can check for a network connection
 309             bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
 310             //If we are not connected retry later
 311             if (!connected)
 312             {
 313                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
 314                 return;
 315             }#1#
 316
 317             try
 318             {
 319                 WorkflowAgent.Start();
 320             }
 321             catch (Exception)
 322             {
 323                 //Faild to authenticate due to network or account error
 324                 //Retry after a while
 325                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
 326             }
 327         }*/
 328
 329         internal class LocalFileComparer:EqualityComparer<CloudAction>
 330         {
 331             public override bool Equals(CloudAction x, CloudAction y)
 332             {
 333                 if (x.Action != y.Action)
 334                     return false;
 335                 if (x.LocalFile != null && y.LocalFile != null && !x.LocalFile.FullName.Equals(y.LocalFile.FullName))
 336                     return false;
 337                 if (x.CloudFile != null && y.CloudFile != null )
 338                 {
 339                     if (x.CloudFile.Hash == null & y.CloudFile.Hash != null)
 340                         return false;
 341                     if (x.CloudFile.Hash != null & y.CloudFile.Hash == null)
 342                         return false;
 343                     if (x.CloudFile.Hash == null & y.CloudFile.Hash == null)
 344                         return (x.CloudFile.Name == y.CloudFile.Name);
 345                     if (!x.CloudFile.Hash.Equals(y.CloudFile.Hash))
 346                         return false;
 347                 }
 348                 if (x.CloudFile == null ^ y.CloudFile == null ||
 349                     x.LocalFile == null ^ y.LocalFile == null)
 350                     return false;
 351                 return true;
 352             }
 353
 354             public override int GetHashCode(CloudAction obj)
 355             {
 356                 var hash1 = (obj.LocalFile == null) ? int.MaxValue : obj.LocalFile.FullName.GetHashCode();
 357                 var hash2 = (obj.CloudFile == null) ? int.MaxValue : (obj.CloudFile.Hash ?? obj.CloudFile.Name??"").GetHashCode();
 358                 var hash3 = obj.Action.GetHashCode();
 359                 return hash1 ^ hash2 & hash3;
 360             }
 361         }
 362
 363         private Timer timer;
 364
 365         private void StartNetworkAgent()
 366         {
 367
 368             NetworkAgent.AddAccount(_accountInfo);
 369
 370             NetworkAgent.StatusNotification = StatusNotification;
 371
 372             NetworkAgent.Start();
 373
 374             PollAgent.StatusNotification = StatusNotification;
 375
 376             PollAgent.PollRemoteFiles();
 377         }
 378
 379         //Make sure a hidden cache folder exists to store partial downloads
 380         private static string CreateHiddenFolder(string rootPath, string folderName)
 381         {
 382             if (String.IsNullOrWhiteSpace(rootPath))
 383                 throw new ArgumentNullException("rootPath");
 384             if (!Path.IsPathRooted(rootPath))
 385                 throw new ArgumentException("rootPath");
 386             if (String.IsNullOrWhiteSpace(folderName))
 387                 throw new ArgumentNullException("folderName");
 388             Contract.EndContractBlock();
 389
 390             var folder = Path.Combine(rootPath, folderName);
 391             if (!Directory.Exists(folder))
 392             {
 393                 var info = Directory.CreateDirectory(folder);
 394                 info.Attributes |= FileAttributes.Hidden;
 395
 396                 Log.InfoFormat("Created cache Folder: {0}", folder);
 397             }
 398             else
 399             {
 400                 var info = new DirectoryInfo(folder);
 401                 if ((info.Attributes & FileAttributes.Hidden) == 0)
 402                 {
 403                     info.Attributes |= FileAttributes.Hidden;
 404                     Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
 405                 }
 406             }
 407             return folder;
 408         }
 409
 410
 411
 412
 413         private void StartWatcherAgent()
 414         {
 415             AgentLocator<FileAgent>.Register(FileAgent,RootPath);
 416
 417             FileAgent.StatusKeeper = StatusKeeper;
 418             FileAgent.Workflow = Workflow;
 419             FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
 420             FileAgent.Start(_accountInfo, RootPath);
 421         }
 422
 423         public void Stop()
 424         {
 425             AgentLocator<FileAgent>.Remove(RootPath);
 426
 427             if (FileAgent!=null)
 428                 FileAgent.Stop();
 429             FileAgent = null;
 430             if (timer != null)
 431                 timer.Dispose();
 432             timer = null;
 433         }
 434
 435
 436         ~PithosMonitor()
 437         {
 438             Dispose(false);
 439         }
 440
 441         public void Dispose()
 442         {
 443             Dispose(true);
 444             GC.SuppressFinalize(this);
 445         }
 446
 447         protected virtual void Dispose(bool disposing)
 448         {
 449             if (disposing)
 450             {
 451                 Stop();
 452             }
 453         }
 454
 455
 456         public void MoveFileStates(string oldPath, string newPath)
 457         {
 458             if (String.IsNullOrWhiteSpace(oldPath))
 459                 throw new ArgumentNullException("oldPath");
 460             if (!Path.IsPathRooted(oldPath))
 461                 throw new ArgumentException("oldPath must be an absolute path","oldPath");
 462             if (string.IsNullOrWhiteSpace(newPath))
 463                 throw new ArgumentNullException("newPath");
 464             if (!Path.IsPathRooted(newPath))
 465                 throw new ArgumentException("newPath must be an absolute path","newPath");
 466             Contract.EndContractBlock();
 467
 468             StatusKeeper.ChangeRoots(oldPath, newPath);
 469         }
 470
 471         public void AddSelectivePaths(string[] added)
 472         {
 473            /* FileAgent.SelectivePaths.AddRange(added);
 474             NetworkAgent.SyncPaths(added);*/
 475         }
 476
 477         public void RemoveSelectivePaths(string[] removed)
 478         {
 479             FileAgent.SelectivePaths.RemoveAll(removed.Contains);
 480             foreach (var removedPath in removed.Where(Directory.Exists))
 481             {
 482                 Directory.Delete(removedPath,true);
 483             }
 484         }
 485
 486         public IEnumerable<ObjectInfo> GetRootFolders()
 487         {
 488             var dirs = from container in CloudClient.ListContainers(UserName)
 489                        from dir in CloudClient.ListObjects(UserName, container.Name, "")
 490                        where dir.Content_Type=="application/directory"
 491                        select dir;
 492             return dirs;
 493         }
 494
 495         public ObjectInfo GetObjectInfo(string filePath)
 496         {
 497             if (String.IsNullOrWhiteSpace(filePath))
 498                 throw new ArgumentNullException("filePath");
 499             Contract.EndContractBlock();
 500
 501             var file=new FileInfo(filePath);
 502             string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
 503             var relativePath = file.AsRelativeTo(RootPath);
 504
 505             string accountName,container;
 506
 507             var parts=relativePath.Split('\\');
 508
 509             var accountInfo = _accountInfo;
 510             if (relativePath.StartsWith(FolderConstants.OthersFolder))
 511             {
 512                 accountName = parts[1];
 513                 container = parts[2];
 514                 relativeUrl = String.Join("/", parts.Splice(3));
 515                 //Create the root URL for the target account
 516                 var oldName = UserName;
 517                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
 518                 var nameIndex=absoluteUri.IndexOf(oldName);
 519                 var root=absoluteUri.Substring(0, nameIndex);
 520
 521                 accountInfo = new AccountInfo
 522                 {
 523                     UserName = accountName,
 524                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 525                     StorageUri = new Uri(root + accountName),
 526                     BlockHash=accountInfo.BlockHash,
 527                     BlockSize=accountInfo.BlockSize,
 528                     Token=accountInfo.Token
 529                 };
 530             }
 531             else
 532             {
 533                 accountName = this.UserName;
 534                 container = parts[0];
 535                 relativeUrl = String.Join("/", parts.Splice(1));
 536             }
 537
 538             var client = new CloudFilesClient(accountInfo);
 539             var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
 540             return objectInfo;
 541         }
 542
 543         public Task<ContainerInfo> GetContainerInfo(string filePath)
 544         {
 545             if (String.IsNullOrWhiteSpace(filePath))
 546                 throw new ArgumentNullException("filePath");
 547             Contract.EndContractBlock();
 548
 549             var file=new FileInfo(filePath);
 550             var relativePath = file.AsRelativeTo(RootPath);
 551
 552             string accountName,container;
 553
 554             var parts=relativePath.Split('\\');
 555
 556             var accountInfo = _accountInfo;
 557             if (relativePath.StartsWith(FolderConstants.OthersFolder))
 558             {
 559                 accountName = parts[1];
 560                 container = parts[2];
 561                 //Create the root URL for the target account
 562                 var oldName = UserName;
 563                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
 564                 var nameIndex=absoluteUri.IndexOf(oldName);
 565                 var root=absoluteUri.Substring(0, nameIndex);
 566
 567                 accountInfo = new AccountInfo
 568                 {
 569                     UserName = accountName,
 570                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
 571                     StorageUri = new Uri(root + accountName),
 572                     BlockHash=accountInfo.BlockHash,
 573                     BlockSize=accountInfo.BlockSize,
 574                     Token=accountInfo.Token
 575                 };
 576             }
 577             else
 578             {
 579                 accountName = UserName;
 580                 container = parts[0];
 581             }
 582
 583             return Task.Factory.StartNew(() =>
 584             {
 585                 var client = new CloudFilesClient(accountInfo);
 586                 var containerInfo = client.GetContainerInfo(accountName, container);
 587                 return containerInfo;
 588             });
 589         }
 590     }
 591 }