#region
/* -----------------------------------------------------------------------
*
*
* Copyright 2011-2012 GRNET S.A. All rights reserved.
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* 1. Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* 2. Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
*
* THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and
* documentation are those of the authors and should not be
* interpreted as representing official policies, either expressed
* or implied, of GRNET S.A.
*
* -----------------------------------------------------------------------
*/
#endregion
using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
using System.Diagnostics.Contracts;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using Pithos.Core.Agents;
using Pithos.Interfaces;
using Pithos.Network;
using log4net;
namespace Pithos.Core
{
[Export(typeof(PithosMonitor))]
public class PithosMonitor:IDisposable
{
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private int _blockSize;
private string _blockHash;
[Import]
public IPithosSettings Settings{get;set;}
private IStatusKeeper _statusKeeper;
[Import]
public IStatusKeeper StatusKeeper
{
get { return _statusKeeper; }
set
{
_statusKeeper = value;
FileAgent.StatusKeeper = value;
}
}
private IPithosWorkflow _workflow;
[Import]
public IPithosWorkflow Workflow
{
get { return _workflow; }
set
{
_workflow = value;
FileAgent.Workflow = value;
}
}
public ICloudClient CloudClient { get; set; }
public IStatusNotification StatusNotification { get; set; }
//[Import]
public FileAgent FileAgent { get; private set; }
private WorkflowAgent _workflowAgent;
[Import]
public WorkflowAgent WorkflowAgent
{
get { return _workflowAgent; }
set
{
_workflowAgent = value;
FileAgent.WorkflowAgent = value;
}
}
[Import]
public NetworkAgent NetworkAgent { get; set; }
[Import]
public PollAgent PollAgent { get; set; }
public string UserName { get; set; }
private string _apiKey;
public string ApiKey
{
get { return _apiKey; }
set
{
_apiKey = value;
if (_accountInfo != null)
_accountInfo.Token = value;
}
}
private AccountInfo _accountInfo;
public bool Pause
{
get { return FileAgent.Pause; }
set
{
FileAgent.Pause = value;
}
}
private string _rootPath;
public string RootPath
{
get { return _rootPath; }
set
{
_rootPath = String.IsNullOrWhiteSpace(value)
? String.Empty
: value.ToLower();
}
}
CancellationTokenSource _cancellationSource;
public PithosMonitor()
{
FileAgent = new FileAgent();
}
private bool _started;
public void Start()
{
if (String.IsNullOrWhiteSpace(ApiKey))
throw new InvalidOperationException("The ApiKey is empty");
if (String.IsNullOrWhiteSpace(UserName))
throw new InvalidOperationException("The UserName is empty");
if (String.IsNullOrWhiteSpace(AuthenticationUrl))
throw new InvalidOperationException("The Authentication url is empty");
Contract.EndContractBlock();
//If the account doesn't have a valid path, don't start monitoring but don't throw either
if (String.IsNullOrWhiteSpace(RootPath))
//TODO; Warn user?
return;
WorkflowAgent.StatusNotification = StatusNotification;
StatusNotification.NotifyChange("Starting");
if (_started)
{
if (!_cancellationSource.IsCancellationRequested)
return;
}
_cancellationSource = new CancellationTokenSource();
CloudClient = new CloudFilesClient(UserName, ApiKey)
{UsePithos = true, AuthenticationUrl = AuthenticationUrl};
_accountInfo = CloudClient.Authenticate();
_accountInfo.SiteUri = AuthenticationUrl;
_accountInfo.AccountPath = RootPath;
var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
if (!Directory.Exists(pithosFolder))
Directory.CreateDirectory(pithosFolder);
//Create the cache folder and ensure it is hidden
CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
var policy=CloudClient.GetAccountPolicies(_accountInfo);
StatusNotification.NotifyAccount(policy);
EnsurePithosContainers();
StatusKeeper.BlockHash = _blockHash;
StatusKeeper.BlockSize = _blockSize;
StatusKeeper.StartProcessing(_cancellationSource.Token);
IndexLocalFiles();
//Extract the URIs from the string collection
var settings = Settings.Accounts.First(s => s.AccountName == _accountInfo.UserName);
var selectiveUrls=settings.SelectiveFolders.Cast().Select(url => new Uri(url)).ToArray();
SetSelectivePaths(selectiveUrls,null,null);
StartWatcherAgent();
StartNetworkAgent();
WorkflowAgent.RestartInterruptedFiles(_accountInfo);
_started = true;
}
private void EnsurePithosContainers()
{
//Create the two default containers if they are missing
var pithosContainers = new List{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
foreach (var container in pithosContainers)
{
var info=CloudClient.GetContainerInfo(UserName, container);
if (info == ContainerInfo.Empty)
{
CloudClient.CreateContainer(UserName, container);
info = CloudClient.GetContainerInfo(UserName, container);
}
_blockSize = info.BlockSize;
_blockHash = info.BlockHash;
_accountInfo.BlockSize = _blockSize;
_accountInfo.BlockHash = _blockHash;
}
}
public string AuthenticationUrl { get; set; }
private void IndexLocalFiles()
{
using (ThreadContext.Stacks["Operation"].Push("Indexing local files"))
{
try
{
//StatusNotification.NotifyChange("Indexing Local Files");
Log.Info("Start local indexing");
StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Indexing Local Files");
var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
var directory = new DirectoryInfo(RootPath);
var files =
from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
!file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
select file;
StatusKeeper.ProcessExistingFiles(files);
}
catch (Exception exc)
{
Log.Error("[ERROR]", exc);
}
finally
{
Log.Info("[END]");
}
StatusNotification.SetPithosStatus(PithosStatus.LocalComplete,"Indexing Completed");
}
}
/* private void StartWorkflowAgent()
{
WorkflowAgent.StatusNotification = StatusNotification;
/* //On Vista and up we can check for a network connection
bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
//If we are not connected retry later
if (!connected)
{
Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
return;
}#1#
try
{
WorkflowAgent.Start();
}
catch (Exception)
{
//Faild to authenticate due to network or account error
//Retry after a while
Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
}
}*/
private void StartNetworkAgent()
{
NetworkAgent.StatusNotification = StatusNotification;
//TODO: The Network and Poll agents are not account specific
//They should be moved outside PithosMonitor
NetworkAgent.Start();
PollAgent.AddAccount(_accountInfo);
PollAgent.StatusNotification = StatusNotification;
PollAgent.PollRemoteFiles();
}
//Make sure a hidden cache folder exists to store partial downloads
private static void CreateHiddenFolder(string rootPath, string folderName)
{
if (String.IsNullOrWhiteSpace(rootPath))
throw new ArgumentNullException("rootPath");
if (!Path.IsPathRooted(rootPath))
throw new ArgumentException("rootPath");
if (String.IsNullOrWhiteSpace(folderName))
throw new ArgumentNullException("folderName");
Contract.EndContractBlock();
var folder = Path.Combine(rootPath, folderName);
if (!Directory.Exists(folder))
{
var info = Directory.CreateDirectory(folder);
info.Attributes |= FileAttributes.Hidden;
Log.InfoFormat("Created cache Folder: {0}", folder);
}
else
{
var info = new DirectoryInfo(folder);
if ((info.Attributes & FileAttributes.Hidden) == 0)
{
info.Attributes |= FileAttributes.Hidden;
Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
}
}
}
private void StartWatcherAgent()
{
if (Log.IsDebugEnabled)
Log.DebugFormat("Start Folder Monitoring [{0}]",RootPath);
AgentLocator.Register(FileAgent,RootPath);
FileAgent.IdleTimeout = Settings.FileIdleTimeout;
FileAgent.StatusKeeper = StatusKeeper;
FileAgent.StatusNotification = StatusNotification;
FileAgent.Workflow = Workflow;
FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
FileAgent.Start(_accountInfo, RootPath);
}
public void Stop()
{
AgentLocator.Remove(RootPath);
if (FileAgent!=null)
FileAgent.Stop();
FileAgent = null;
}
~PithosMonitor()
{
Dispose(false);
}
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
Stop();
}
}
public void MoveFileStates(string oldPath, string newPath)
{
if (String.IsNullOrWhiteSpace(oldPath))
throw new ArgumentNullException("oldPath");
if (!Path.IsPathRooted(oldPath))
throw new ArgumentException("oldPath must be an absolute path","oldPath");
if (string.IsNullOrWhiteSpace(newPath))
throw new ArgumentNullException("newPath");
if (!Path.IsPathRooted(newPath))
throw new ArgumentException("newPath must be an absolute path","newPath");
Contract.EndContractBlock();
StatusKeeper.ChangeRoots(oldPath, newPath);
}
public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed)
{
//Convert the uris to paths
var selectivePaths = UrisToFilePaths(uris);
FileAgent.SelectivePaths=selectivePaths;
PollAgent.SetSyncUris(uris);
var removedPaths = UrisToFilePaths(removed);
UnversionSelectivePaths(removedPaths);
}
///
/// Mark all unselected paths as Unversioned
///
///
private void UnversionSelectivePaths(List removed)
{
if (removed == null)
return;
//Ensure we remove any file state below the deleted folders
FileState.UnversionPaths(removed);
}
///
/// Return a list of absolute filepaths from a list of Uris
///
///
///
private List UrisToFilePaths(IEnumerable uris)
{
if (uris == null)
return new List();
return (from uri in uris
let relativePath = _accountInfo.StorageUri
.MakeRelativeUri(uri)
.RelativeUriToFilePath()
//Trim the account name
select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList();
}
public ObjectInfo GetObjectInfo(string filePath)
{
if (String.IsNullOrWhiteSpace(filePath))
throw new ArgumentNullException("filePath");
Contract.EndContractBlock();
var file=new FileInfo(filePath);
string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
var relativePath = file.AsRelativeTo(RootPath);
string accountName,container;
var parts=relativePath.Split('\\');
var accountInfo = _accountInfo;
if (relativePath.StartsWith(FolderConstants.OthersFolder))
{
accountName = parts[1];
container = parts[2];
relativeUrl = String.Join("/", parts.Splice(3));
//Create the root URL for the target account
var oldName = UserName;
var absoluteUri = _accountInfo.StorageUri.AbsoluteUri;
var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
var root=absoluteUri.Substring(0, nameIndex);
accountInfo = new AccountInfo
{
UserName = accountName,
AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
StorageUri = new Uri(root + accountName),
BlockHash=accountInfo.BlockHash,
BlockSize=accountInfo.BlockSize,
Token=accountInfo.Token
};
}
else
{
accountName = UserName;
container = parts[0];
relativeUrl = String.Join("/", parts.Splice(1));
}
var client = new CloudFilesClient(accountInfo);
var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
return objectInfo;
}
public Task GetContainerInfo(string filePath)
{
if (String.IsNullOrWhiteSpace(filePath))
throw new ArgumentNullException("filePath");
Contract.EndContractBlock();
var file=new FileInfo(filePath);
var relativePath = file.AsRelativeTo(RootPath);
string accountName,container;
var parts=relativePath.Split('\\');
var accountInfo = _accountInfo;
if (relativePath.StartsWith(FolderConstants.OthersFolder))
{
accountName = parts[1];
container = parts[2];
//Create the root URL for the target account
var oldName = UserName;
var absoluteUri = _accountInfo.StorageUri.AbsoluteUri;
var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
var root=absoluteUri.Substring(0, nameIndex);
accountInfo = new AccountInfo
{
UserName = accountName,
AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
StorageUri = new Uri(root + accountName),
BlockHash=accountInfo.BlockHash,
BlockSize=accountInfo.BlockSize,
Token=accountInfo.Token
};
}
else
{
accountName = UserName;
container = parts[0];
}
return Task.Factory.StartNew(() =>
{
var client = new CloudFilesClient(accountInfo);
var containerInfo = client.GetContainerInfo(accountName, container);
return containerInfo;
});
}
}
}