#region /* ----------------------------------------------------------------------- * * * Copyright 2011-2012 GRNET S.A. All rights reserved. * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * 1. Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * 2. Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * The views and conclusions contained in the software and * documentation are those of the authors and should not be * interpreted as representing official policies, either expressed * or implied, of GRNET S.A. * * ----------------------------------------------------------------------- */ #endregion using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel.Composition; using System.Diagnostics; using System.Diagnostics.Contracts; using System.IO; using System.Linq; using System.Net; using System.Net.NetworkInformation; using System.Security.Cryptography; using System.ServiceModel.Description; using System.Text; using System.Threading; using System.Threading.Tasks; using Castle.ActiveRecord.Queries; using Microsoft.WindowsAPICodePack.Net; using Pithos.Core.Agents; using Pithos.Interfaces; using System.ServiceModel; using Pithos.Network; using log4net; namespace Pithos.Core { [Export(typeof(PithosMonitor))] public class PithosMonitor:IDisposable { private int _blockSize; private string _blockHash; [Import] public IPithosSettings Settings{get;set;} private IStatusKeeper _statusKeeper; [Import] public IStatusKeeper StatusKeeper { get { return _statusKeeper; } set { _statusKeeper = value; FileAgent.StatusKeeper = value; } } private IPithosWorkflow _workflow; [Import] public IPithosWorkflow Workflow { get { return _workflow; } set { _workflow = value; FileAgent.Workflow = value; } } public ICloudClient CloudClient { get; set; } public IStatusNotification StatusNotification { get; set; } //[Import] public FileAgent FileAgent { get; private set; } private WorkflowAgent _workflowAgent; [Import] public WorkflowAgent WorkflowAgent { get { return _workflowAgent; } set { _workflowAgent = value; FileAgent.WorkflowAgent = value; } } [Import] public NetworkAgent NetworkAgent { get; set; } [Import] public PollAgent PollAgent { get; set; } public string UserName { get; set; } private string _apiKey; public string ApiKey { get { return _apiKey; } set { _apiKey = value; if (_accountInfo != null) _accountInfo.Token = value; } } private AccountInfo _accountInfo; private static readonly ILog Log = LogManager.GetLogger(typeof(PithosMonitor)); public bool Pause { get { return FileAgent.Pause; } set { FileAgent.Pause = value; if (value) { StatusKeeper.SetPithosStatus(PithosStatus.SyncPaused); StatusNotification.NotifyChange("Paused"); } else { StatusKeeper.SetPithosStatus(PithosStatus.InSynch); StatusNotification.NotifyChange("Synchronizing"); } } } private string _rootPath; public string RootPath { get { return _rootPath; } set { _rootPath = String.IsNullOrWhiteSpace(value) ? String.Empty : value.ToLower(); } } CancellationTokenSource _cancellationSource; public PithosMonitor() { FileAgent = new FileAgent(); } private bool _started; public void Start() { if (String.IsNullOrWhiteSpace(ApiKey)) throw new InvalidOperationException("The ApiKey is empty"); if (String.IsNullOrWhiteSpace(UserName)) throw new InvalidOperationException("The UserName is empty"); if (String.IsNullOrWhiteSpace(AuthenticationUrl)) throw new InvalidOperationException("The Authentication url is empty"); Contract.EndContractBlock(); //If the account doesn't have a valid path, don't start monitoring but don't throw either if (String.IsNullOrWhiteSpace(RootPath)) //TODO; Warn user? return; WorkflowAgent.StatusNotification = StatusNotification; StatusNotification.NotifyChange("Starting"); if (_started) { if (!_cancellationSource.IsCancellationRequested) return; } _cancellationSource = new CancellationTokenSource(); CloudClient=new CloudFilesClient(UserName,ApiKey); CloudClient.UsePithos = true; CloudClient.AuthenticationUrl = this.AuthenticationUrl; _accountInfo = CloudClient.Authenticate(); _accountInfo.SiteUri = AuthenticationUrl; _accountInfo.AccountPath = RootPath; var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer); if (!Directory.Exists(pithosFolder)) Directory.CreateDirectory(pithosFolder); //Create the cache folder and ensure it is hidden CreateHiddenFolder(RootPath, FolderConstants.CacheFolder); var policy=CloudClient.GetAccountPolicies(_accountInfo); StatusNotification.NotifyAccount(policy); EnsurePithosContainers(); StatusKeeper.BlockHash = _blockHash; StatusKeeper.BlockSize = _blockSize; StatusKeeper.StartProcessing(_cancellationSource.Token); IndexLocalFiles(); //Extract the URIs from the string collection var settings = Settings.Accounts.First(s => s.AccountName == _accountInfo.UserName); var selectiveUrls=settings.SelectiveFolders.Cast().Select(url => new Uri(url)).ToArray(); SetSelectivePaths(selectiveUrls,null,null); StartWatcherAgent(); StartNetworkAgent(); WorkflowAgent.RestartInterruptedFiles(_accountInfo); _started = true; } private void EnsurePithosContainers() { //Create the two default containers if they are missing var pithosContainers = new List{ FolderConstants.TrashContainer,FolderConstants.PithosContainer}; foreach (var container in pithosContainers) { var info=CloudClient.GetContainerInfo(this.UserName, container); if (info == ContainerInfo.Empty) { CloudClient.CreateContainer(this.UserName, container); info = CloudClient.GetContainerInfo(this.UserName, container); } _blockSize = info.BlockSize; _blockHash = info.BlockHash; _accountInfo.BlockSize = _blockSize; _accountInfo.BlockHash = _blockHash; } } public string AuthenticationUrl { get; set; } private void IndexLocalFiles() { StatusNotification.NotifyChange("Indexing Local Files",TraceLevel.Info); using (log4net.ThreadContext.Stacks["Monitor"].Push("Indexing local files")) { Log.Info("START"); try { var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder); var directory = new DirectoryInfo(RootPath); var files = from file in directory.EnumerateFiles("*", SearchOption.AllDirectories) where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) && !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase) select file; StatusKeeper.ProcessExistingFiles(files); } catch (Exception exc) { Log.Error("[ERROR]", exc); } finally { Log.Info("[END]"); } } } /* private void StartWorkflowAgent() { WorkflowAgent.StatusNotification = StatusNotification; /* //On Vista and up we can check for a network connection bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet; //If we are not connected retry later if (!connected) { Task.Factory.StartNewDelayed(10000, StartWorkflowAgent); return; }#1# try { WorkflowAgent.Start(); } catch (Exception) { //Faild to authenticate due to network or account error //Retry after a while Task.Factory.StartNewDelayed(10000, StartWorkflowAgent); } }*/ internal class LocalFileComparer:EqualityComparer { public override bool Equals(CloudAction x, CloudAction y) { if (x.Action != y.Action) return false; if (x.LocalFile != null && y.LocalFile != null && !x.LocalFile.FullName.Equals(y.LocalFile.FullName)) return false; if (x.CloudFile != null && y.CloudFile != null ) { if (x.CloudFile.Hash == null & y.CloudFile.Hash != null) return false; if (x.CloudFile.Hash != null & y.CloudFile.Hash == null) return false; if (x.CloudFile.Hash == null & y.CloudFile.Hash == null) return (x.CloudFile.Name == y.CloudFile.Name); if (!x.CloudFile.Hash.Equals(y.CloudFile.Hash)) return false; } if (x.CloudFile == null ^ y.CloudFile == null || x.LocalFile == null ^ y.LocalFile == null) return false; return true; } public override int GetHashCode(CloudAction obj) { var hash1 = (obj.LocalFile == null) ? int.MaxValue : obj.LocalFile.FullName.GetHashCode(); var hash2 = (obj.CloudFile == null) ? int.MaxValue : (obj.CloudFile.Hash ?? obj.CloudFile.Name??"").GetHashCode(); var hash3 = obj.Action.GetHashCode(); return hash1 ^ hash2 & hash3; } } private Timer timer; private void StartNetworkAgent() { NetworkAgent.AddAccount(_accountInfo); NetworkAgent.StatusNotification = StatusNotification; NetworkAgent.Start(); PollAgent.AddAccount(_accountInfo); PollAgent.StatusNotification = StatusNotification; PollAgent.PollRemoteFiles(); } //Make sure a hidden cache folder exists to store partial downloads private static string CreateHiddenFolder(string rootPath, string folderName) { if (String.IsNullOrWhiteSpace(rootPath)) throw new ArgumentNullException("rootPath"); if (!Path.IsPathRooted(rootPath)) throw new ArgumentException("rootPath"); if (String.IsNullOrWhiteSpace(folderName)) throw new ArgumentNullException("folderName"); Contract.EndContractBlock(); var folder = Path.Combine(rootPath, folderName); if (!Directory.Exists(folder)) { var info = Directory.CreateDirectory(folder); info.Attributes |= FileAttributes.Hidden; Log.InfoFormat("Created cache Folder: {0}", folder); } else { var info = new DirectoryInfo(folder); if ((info.Attributes & FileAttributes.Hidden) == 0) { info.Attributes |= FileAttributes.Hidden; Log.InfoFormat("Reset cache folder to hidden: {0}", folder); } } return folder; } private void StartWatcherAgent() { AgentLocator.Register(FileAgent,RootPath); FileAgent.StatusKeeper = StatusKeeper; FileAgent.Workflow = Workflow; FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder); FileAgent.Start(_accountInfo, RootPath); } public void Stop() { AgentLocator.Remove(RootPath); if (FileAgent!=null) FileAgent.Stop(); FileAgent = null; if (timer != null) timer.Dispose(); timer = null; } ~PithosMonitor() { Dispose(false); } public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } protected virtual void Dispose(bool disposing) { if (disposing) { Stop(); } } public void MoveFileStates(string oldPath, string newPath) { if (String.IsNullOrWhiteSpace(oldPath)) throw new ArgumentNullException("oldPath"); if (!Path.IsPathRooted(oldPath)) throw new ArgumentException("oldPath must be an absolute path","oldPath"); if (string.IsNullOrWhiteSpace(newPath)) throw new ArgumentNullException("newPath"); if (!Path.IsPathRooted(newPath)) throw new ArgumentException("newPath must be an absolute path","newPath"); Contract.EndContractBlock(); StatusKeeper.ChangeRoots(oldPath, newPath); } public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed) { //Convert the uris to paths var selectivePaths = UrisToFilePaths(uris); FileAgent.SelectivePaths=selectivePaths; PollAgent.SetSyncUris(uris); var removedPaths = UrisToFilePaths(removed); RemoveSelectivePaths(removedPaths); } /// /// Return a list of absolute filepaths from a list of Uris /// /// /// private List UrisToFilePaths(IEnumerable uris) { if (uris == null) return new List(); return (from uri in uris let relativePath = _accountInfo.StorageUri .MakeRelativeUri(uri) .RelativeUriToFilePath() //Trim the account name select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList(); } /// /// Delete the folders that were removed from synchronization /// /// private void RemoveSelectivePaths(List removed) { if (removed == null) return; foreach (var removedPath in removed.Where(Directory.Exists)) { try { Directory.Delete(removedPath, true); } catch { } } //Ensure we remove any file state below the deleted folders FileState.RemovePaths(removed); } public ObjectInfo GetObjectInfo(string filePath) { if (String.IsNullOrWhiteSpace(filePath)) throw new ArgumentNullException("filePath"); Contract.EndContractBlock(); var file=new FileInfo(filePath); string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath); var relativePath = file.AsRelativeTo(RootPath); string accountName,container; var parts=relativePath.Split('\\'); var accountInfo = _accountInfo; if (relativePath.StartsWith(FolderConstants.OthersFolder)) { accountName = parts[1]; container = parts[2]; relativeUrl = String.Join("/", parts.Splice(3)); //Create the root URL for the target account var oldName = UserName; var absoluteUri = _accountInfo.StorageUri.AbsoluteUri; var nameIndex=absoluteUri.IndexOf(oldName); var root=absoluteUri.Substring(0, nameIndex); accountInfo = new AccountInfo { UserName = accountName, AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]), StorageUri = new Uri(root + accountName), BlockHash=accountInfo.BlockHash, BlockSize=accountInfo.BlockSize, Token=accountInfo.Token }; } else { accountName = this.UserName; container = parts[0]; relativeUrl = String.Join("/", parts.Splice(1)); } var client = new CloudFilesClient(accountInfo); var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl); return objectInfo; } public Task GetContainerInfo(string filePath) { if (String.IsNullOrWhiteSpace(filePath)) throw new ArgumentNullException("filePath"); Contract.EndContractBlock(); var file=new FileInfo(filePath); var relativePath = file.AsRelativeTo(RootPath); string accountName,container; var parts=relativePath.Split('\\'); var accountInfo = _accountInfo; if (relativePath.StartsWith(FolderConstants.OthersFolder)) { accountName = parts[1]; container = parts[2]; //Create the root URL for the target account var oldName = UserName; var absoluteUri = _accountInfo.StorageUri.AbsoluteUri; var nameIndex=absoluteUri.IndexOf(oldName); var root=absoluteUri.Substring(0, nameIndex); accountInfo = new AccountInfo { UserName = accountName, AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]), StorageUri = new Uri(root + accountName), BlockHash=accountInfo.BlockHash, BlockSize=accountInfo.BlockSize, Token=accountInfo.Token }; } else { accountName = UserName; container = parts[0]; } return Task.Factory.StartNew(() => { var client = new CloudFilesClient(accountInfo); var containerInfo = client.GetContainerInfo(accountName, container); return containerInfo; }); } } }