using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel.Composition; using System.Diagnostics; using System.Diagnostics.Contracts; using System.IO; using System.Linq; using System.Net.NetworkInformation; using System.Security.Cryptography; using System.ServiceModel.Description; using System.Text; using System.Threading; using System.Threading.Tasks; using Castle.ActiveRecord.Queries; using Microsoft.WindowsAPICodePack.Net; using Pithos.Core.Agents; using Pithos.Interfaces; using System.ServiceModel; using Pithos.Network; using log4net; namespace Pithos.Core { [Export(typeof(PithosMonitor))] public class PithosMonitor:IDisposable { private int _blockSize; private string _blockHash; [Import] public IPithosSettings Settings{get;set;} private IStatusKeeper _statusKeeper; [Import] public IStatusKeeper StatusKeeper { get { return _statusKeeper; } set { _statusKeeper = value; FileAgent.StatusKeeper = value; } } private IPithosWorkflow _workflow; [Import] public IPithosWorkflow Workflow { get { return _workflow; } set { _workflow = value; FileAgent.Workflow = value; } } public ICloudClient CloudClient { get; set; } public IStatusNotification StatusNotification { get; set; } //[Import] public FileAgent FileAgent { get; private set; } private WorkflowAgent _workflowAgent; [Import] public WorkflowAgent WorkflowAgent { get { return _workflowAgent; } set { _workflowAgent = value; FileAgent.WorkflowAgent = value; } } [Import] public NetworkAgent NetworkAgent { get; set; } public string UserName { get; set; } private string _apiKey; public string ApiKey { get { return _apiKey; } set { _apiKey = value; if (_accountInfo != null) _accountInfo.Token = value; } } private AccountInfo _accountInfo; private static readonly ILog Log = LogManager.GetLogger(typeof(PithosMonitor)); public bool Pause { get { return FileAgent.Pause; } set { FileAgent.Pause = value; if (value) { StatusKeeper.SetPithosStatus(PithosStatus.SyncPaused); StatusNotification.NotifyChange("Paused"); } else { StatusKeeper.SetPithosStatus(PithosStatus.InSynch); StatusNotification.NotifyChange("Synchronizing"); } } } private string _rootPath; public string RootPath { get { return _rootPath; } set { _rootPath = String.IsNullOrWhiteSpace(value) ? String.Empty : value.ToLower(); } } CancellationTokenSource _cancellationSource; public PithosMonitor() { FileAgent = new FileAgent(); } private bool _started; public void Start() { if (String.IsNullOrWhiteSpace(ApiKey)) throw new InvalidOperationException("The ApiKey is empty"); if (String.IsNullOrWhiteSpace(UserName)) throw new InvalidOperationException("The UserName is empty"); if (String.IsNullOrWhiteSpace(AuthenticationUrl)) throw new InvalidOperationException("The Authentication url is empty"); Contract.EndContractBlock(); //If the account doesn't have a valid path, don't start monitoring but don't throw either if (String.IsNullOrWhiteSpace(RootPath)) //TODO; Warn user? return; StatusNotification.NotifyChange("Starting"); if (_started) { if (!_cancellationSource.IsCancellationRequested) return; } _cancellationSource = new CancellationTokenSource(); CloudClient=new CloudFilesClient(UserName,ApiKey); var proxyUri = ProxyFromSettings(); CloudClient.Proxy = proxyUri; CloudClient.UsePithos = true; CloudClient.AuthenticationUrl = this.AuthenticationUrl; _accountInfo = CloudClient.Authenticate(); _accountInfo.SiteUri = AuthenticationUrl; _accountInfo.AccountPath = RootPath; var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer); if (!Directory.Exists(pithosFolder)) Directory.CreateDirectory(pithosFolder); //Create the cache folder and ensure it is hidden CreateHiddenFolder(RootPath, FolderConstants.CacheFolder); var policy=CloudClient.GetAccountPolicies(_accountInfo); StatusNotification.NotifyAccount(policy); EnsurePithosContainers(); StatusKeeper.BlockHash = _blockHash; StatusKeeper.BlockSize = _blockSize; StatusKeeper.StartProcessing(_cancellationSource.Token); IndexLocalFiles(); StartWatcherAgent(); StartNetworkAgent(); StartWorkflowAgent(); WorkflowAgent.RestartInterruptedFiles(_accountInfo); _started = true; } private void EnsurePithosContainers() { //Create the two default containers if they are missing var pithosContainers = new List{ FolderConstants.TrashContainer,FolderConstants.PithosContainer}; foreach (var container in pithosContainers) { var info=CloudClient.GetContainerInfo(this.UserName, container); if (info == ContainerInfo.Empty) { CloudClient.CreateContainer(this.UserName, container); info = CloudClient.GetContainerInfo(this.UserName, container); } _blockSize = info.BlockSize; _blockHash = info.BlockHash; _accountInfo.BlockSize = _blockSize; _accountInfo.BlockHash = _blockHash; } } public string AuthenticationUrl { get; set; } private Uri ProxyFromSettings() { if (Settings.UseManualProxy) { var proxyUri = new UriBuilder { Host = Settings.ProxyServer, Port = Settings.ProxyPort }; if (Settings.ProxyAuthentication) { proxyUri.UserName = Settings.ProxyUsername; proxyUri.Password = Settings.ProxyPassword; } return proxyUri.Uri; } return null; } private void IndexLocalFiles() { StatusNotification.NotifyChange("Indexing Local Files",TraceLevel.Info); using (log4net.ThreadContext.Stacks["Monitor"].Push("Indexing local files")) { Log.Info("START"); try { var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder); var directory = new DirectoryInfo(RootPath); var files = from file in directory.EnumerateFiles("*", SearchOption.AllDirectories) where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) && !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase) select file; StatusKeeper.ProcessExistingFiles(files); } catch (Exception exc) { Log.Error("[ERROR]", exc); } finally { Log.Info("[END]"); } } } private void StartWorkflowAgent() { //On Vista and up we can check for a network connection bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet; //If we are not connected retry later if (!connected) { Task.Factory.StartNewDelayed(10000, StartWorkflowAgent); return; } try { WorkflowAgent.StatusNotification = StatusNotification; WorkflowAgent.Start(); } catch (Exception) { //Faild to authenticate due to network or account error //Retry after a while Task.Factory.StartNewDelayed(10000, StartWorkflowAgent); } } internal class LocalFileComparer:EqualityComparer { public override bool Equals(CloudAction x, CloudAction y) { if (x.Action != y.Action) return false; if (x.LocalFile != null && y.LocalFile != null && !x.LocalFile.FullName.Equals(y.LocalFile.FullName)) return false; if (x.CloudFile != null && y.CloudFile != null ) { if (x.CloudFile.Hash == null & y.CloudFile.Hash != null) return false; if (x.CloudFile.Hash != null & y.CloudFile.Hash == null) return false; if (x.CloudFile.Hash == null & y.CloudFile.Hash == null) return (x.CloudFile.Name == y.CloudFile.Name); if (!x.CloudFile.Hash.Equals(y.CloudFile.Hash)) return false; } if (x.CloudFile == null ^ y.CloudFile == null || x.LocalFile == null ^ y.LocalFile == null) return false; return true; } public override int GetHashCode(CloudAction obj) { var hash1 = (obj.LocalFile == null) ? int.MaxValue : obj.LocalFile.FullName.GetHashCode(); var hash2 = (obj.CloudFile == null) ? int.MaxValue : (obj.CloudFile.Hash ?? obj.CloudFile.Name??"").GetHashCode(); var hash3 = obj.Action.GetHashCode(); return hash1 ^ hash2 & hash3; } } private Timer timer; private void StartNetworkAgent() { NetworkAgent.AddAccount(_accountInfo); NetworkAgent.StatusNotification = StatusNotification; NetworkAgent.Start(); NetworkAgent.PollRemoteFiles(); } //Make sure a hidden cache folder exists to store partial downloads private static string CreateHiddenFolder(string rootPath, string folderName) { if (String.IsNullOrWhiteSpace(rootPath)) throw new ArgumentNullException("rootPath"); if (!Path.IsPathRooted(rootPath)) throw new ArgumentException("rootPath"); if (String.IsNullOrWhiteSpace(folderName)) throw new ArgumentNullException("folderName"); Contract.EndContractBlock(); var folder = Path.Combine(rootPath, folderName); if (!Directory.Exists(folder)) { var info = Directory.CreateDirectory(folder); info.Attributes |= FileAttributes.Hidden; Log.InfoFormat("Created cache Folder: {0}", folder); } else { var info = new DirectoryInfo(folder); if ((info.Attributes & FileAttributes.Hidden) == 0) { info.Attributes |= FileAttributes.Hidden; Log.InfoFormat("Reset cache folder to hidden: {0}", folder); } } return folder; } private void StartWatcherAgent() { AgentLocator.Register(FileAgent,RootPath); FileAgent.StatusKeeper = StatusKeeper; FileAgent.Workflow = Workflow; FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder); FileAgent.Start(_accountInfo, RootPath); } public void Stop() { AgentLocator.Remove(RootPath); if (FileAgent!=null) FileAgent.Stop(); FileAgent = null; if (timer != null) timer.Dispose(); timer = null; } ~PithosMonitor() { Dispose(false); } public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } protected virtual void Dispose(bool disposing) { if (disposing) { Stop(); } } public void MoveFileStates(string oldPath, string newPath) { if (String.IsNullOrWhiteSpace(oldPath)) throw new ArgumentNullException("oldPath"); if (!Path.IsPathRooted(oldPath)) throw new ArgumentException("oldPath must be an absolute path","oldPath"); if (string.IsNullOrWhiteSpace(newPath)) throw new ArgumentNullException("newPath"); if (!Path.IsPathRooted(newPath)) throw new ArgumentException("newPath must be an absolute path","newPath"); Contract.EndContractBlock(); StatusKeeper.ChangeRoots(oldPath, newPath); } public void AddSelectivePaths(string[] added) { /* FileAgent.SelectivePaths.AddRange(added); NetworkAgent.SyncPaths(added);*/ } public void RemoveSelectivePaths(string[] removed) { FileAgent.SelectivePaths.RemoveAll(removed.Contains); foreach (var removedPath in removed.Where(Directory.Exists)) { Directory.Delete(removedPath,true); } } public IEnumerable GetRootFolders() { var dirs = from container in CloudClient.ListContainers(UserName) from dir in CloudClient.ListObjects(UserName, container.Name, "") select dir.Name; return dirs; } public ObjectInfo GetObjectInfo(string filePath) { if (String.IsNullOrWhiteSpace(filePath)) throw new ArgumentNullException("filePath"); Contract.EndContractBlock(); var file=new FileInfo(filePath); string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath); var relativePath = file.AsRelativeTo(RootPath); string accountName,container; var parts=relativePath.Split('\\'); var accountInfo = _accountInfo; if (relativePath.StartsWith(FolderConstants.OthersFolder)) { accountName = parts[1]; container = parts[2]; relativeUrl = String.Join("/", parts.Splice(3)); //Create the root URL for the target account var oldName = UserName; var absoluteUri = _accountInfo.StorageUri.AbsoluteUri; var nameIndex=absoluteUri.IndexOf(oldName); var root=absoluteUri.Substring(0, nameIndex); accountInfo = new AccountInfo { UserName = accountName, AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]), StorageUri = new Uri(root + accountName), BlockHash=accountInfo.BlockHash, BlockSize=accountInfo.BlockSize, Token=accountInfo.Token }; } else { accountName = this.UserName; container = parts[0]; relativeUrl = String.Join("/", parts.Splice(1)); } var client = new CloudFilesClient(accountInfo); var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl); return objectInfo; } public Task GetContainerInfo(string filePath) { if (String.IsNullOrWhiteSpace(filePath)) throw new ArgumentNullException("filePath"); Contract.EndContractBlock(); var file=new FileInfo(filePath); var relativePath = file.AsRelativeTo(RootPath); string accountName,container; var parts=relativePath.Split('\\'); var accountInfo = _accountInfo; if (relativePath.StartsWith(FolderConstants.OthersFolder)) { accountName = parts[1]; container = parts[2]; //Create the root URL for the target account var oldName = UserName; var absoluteUri = _accountInfo.StorageUri.AbsoluteUri; var nameIndex=absoluteUri.IndexOf(oldName); var root=absoluteUri.Substring(0, nameIndex); accountInfo = new AccountInfo { UserName = accountName, AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]), StorageUri = new Uri(root + accountName), BlockHash=accountInfo.BlockHash, BlockSize=accountInfo.BlockSize, Token=accountInfo.Token }; } else { accountName = UserName; container = parts[0]; } return Task.Factory.StartNew(() => { var client = new CloudFilesClient(accountInfo); var containerInfo = client.GetContainerInfo(accountName, container); return containerInfo; }); } } public interface IStatusNotification { void NotifyChange(string status,TraceLevel level=TraceLevel.Info); void NotifyChangedFile(string filePath); void NotifyAccount(AccountInfo policy); } }