2 /* -----------------------------------------------------------------------
3 * <copyright file="PithosMonitor.cs" company="GRNet">
5 * Copyright 2011-2012 GRNET S.A. All rights reserved.
7 * Redistribution and use in source and binary forms, with or
8 * without modification, are permitted provided that the following
11 * 1. Redistributions of source code must retain the above
12 * copyright notice, this list of conditions and the following
15 * 2. Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer in the documentation and/or other materials
18 * provided with the distribution.
21 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
34 * The views and conclusions contained in the software and
35 * documentation are those of the authors and should not be
36 * interpreted as representing official policies, either expressed
37 * or implied, of GRNET S.A.
39 * -----------------------------------------------------------------------
43 using System.Collections.Generic;
44 using System.ComponentModel.Composition;
45 using System.Diagnostics.Contracts;
48 using System.Reflection;
49 using System.Threading;
50 using System.Threading.Tasks;
51 using Pithos.Core.Agents;
52 using Pithos.Interfaces;
58 [Export(typeof(PithosMonitor))]
59 public class PithosMonitor:IDisposable
61 private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
63 private int _blockSize;
64 private string _blockHash;
67 public IPithosSettings Settings{get;set;}
69 private IStatusKeeper _statusKeeper;
72 public IStatusKeeper StatusKeeper
74 get { return _statusKeeper; }
77 _statusKeeper = value;
78 FileAgent.StatusKeeper = value;
85 private IPithosWorkflow _workflow;
88 public IPithosWorkflow Workflow
90 get { return _workflow; }
94 FileAgent.Workflow = value;
98 public ICloudClient CloudClient { get; set; }
100 public IStatusNotification StatusNotification { get; set; }
103 public FileAgent FileAgent { get; private set; }
105 private WorkflowAgent _workflowAgent;
108 public WorkflowAgent WorkflowAgent
110 get { return _workflowAgent; }
113 _workflowAgent = value;
114 //FileAgent.WorkflowAgent = value;
119 public NetworkAgent NetworkAgent { get; set; }
121 private PollAgent _pollAgent;
124 public PollAgent PollAgent
126 get { return _pollAgent; }
130 FileAgent.PollAgent = value;
134 private Selectives _selectives;
137 public Selectives Selectives
139 get { return _selectives; }
143 FileAgent.Selectives = value;
147 public string UserName { get; set; }
148 private string _apiKey;
151 get { return _apiKey; }
155 if (_accountInfo != null)
156 _accountInfo.Token = value;
160 private AccountInfo _accountInfo;
162 public AccountInfo Account
164 get { return _accountInfo; }
171 public bool Pause { get; set; }
174 get { return FileAgent.Pause; }
177 FileAgent.Pause = value;
181 private string _rootPath;
182 public string RootPath
184 get { return _rootPath; }
187 _rootPath = String.IsNullOrWhiteSpace(value)
194 CancellationTokenSource _cancellationSource;
196 public PithosMonitor()
198 FileAgent = new FileAgent();
200 private bool _started;
204 if (String.IsNullOrWhiteSpace(ApiKey))
205 throw new InvalidOperationException("The ApiKey is empty");
206 if (String.IsNullOrWhiteSpace(UserName))
207 throw new InvalidOperationException("The UserName is empty");
208 if (String.IsNullOrWhiteSpace(AuthenticationUrl))
209 throw new InvalidOperationException("The Authentication url is empty");
210 Contract.EndContractBlock();
212 //If the account doesn't have a valid path, don't start monitoring but don't throw either
213 if (String.IsNullOrWhiteSpace(RootPath))
217 WorkflowAgent.StatusNotification = StatusNotification;
219 StatusNotification.NotifyChange("Starting");
222 if (!_cancellationSource.IsCancellationRequested)
225 _cancellationSource = new CancellationTokenSource();
229 CloudClient = new CloudFilesClient(UserName, ApiKey)
230 {UsePithos = true, AuthenticationUrl = AuthenticationUrl};
231 _accountInfo = CloudClient.Authenticate();
233 _accountInfo.SiteUri = AuthenticationUrl;
234 _accountInfo.AccountPath = RootPath;
237 var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
238 if (!Directory.Exists(pithosFolder))
239 Directory.CreateDirectory(pithosFolder);
240 //Create the cache folder and ensure it is hidden
241 CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
243 var policy=CloudClient.GetAccountPolicies(_accountInfo);
245 StatusNotification.NotifyAccount(policy);
246 EnsurePithosContainers();
248 StatusKeeper.BlockHash = _blockHash;
249 StatusKeeper.BlockSize = _blockSize;
251 StatusKeeper.StartProcessing(_cancellationSource.Token);
253 //Extract the URIs from the string collection
254 var settings = Settings.Accounts.First(s => s.AccountKey == _accountInfo.AccountKey );
256 var selectiveUrls=settings.SelectiveFolders.Cast<string>().Select(url => new Uri(url,UriKind.RelativeOrAbsolute))
257 .Where(uri=>uri.IsAbsoluteUri).ToArray();
259 SetSelectivePaths(selectiveUrls,null,null);
265 WorkflowAgent.RestartInterruptedFiles(_accountInfo);
269 private void EnsurePithosContainers()
272 //Create the two default containers if they are missing
273 var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
274 foreach (var container in pithosContainers)
276 var info=CloudClient.GetContainerInfo(UserName, container);
277 if (info == ContainerInfo.Empty)
279 CloudClient.CreateContainer(UserName, container);
280 info = CloudClient.GetContainerInfo(UserName, container);
282 _blockSize = info.BlockSize;
283 _blockHash = info.BlockHash;
284 _accountInfo.BlockSize = _blockSize;
285 _accountInfo.BlockHash = _blockHash;
289 public string AuthenticationUrl { get; set; }
291 private void IndexLocalFiles()
293 using (ThreadContext.Stacks["Operation"].Push("Indexing local files"))
298 //StatusNotification.NotifyChange("Indexing Local Files");
299 Log.Info("Start local indexing");
300 StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Indexing Local Files");
302 var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
303 var directory = new DirectoryInfo(RootPath);
305 from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
306 where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
307 !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
309 StatusKeeper.ProcessExistingFiles(files);
312 catch (Exception exc)
314 Log.Error("[ERROR]", exc);
320 StatusNotification.SetPithosStatus(PithosStatus.LocalComplete,"Indexing Completed");
328 /* private void StartWorkflowAgent()
330 WorkflowAgent.StatusNotification = StatusNotification;
332 /* //On Vista and up we can check for a network connection
333 bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
334 //If we are not connected retry later
337 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
343 WorkflowAgent.Start();
347 //Faild to authenticate due to network or account error
348 //Retry after a while
349 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
354 private void StartNetworkAgent()
356 NetworkAgent.StatusNotification = StatusNotification;
358 //TODO: The Network and Poll agents are not account specific
359 //They should be moved outside PithosMonitor
360 NetworkAgent.Start();
362 PollAgent.AddAccount(_accountInfo);
364 PollAgent.StatusNotification = StatusNotification;
366 PollAgent.PollRemoteFiles();
369 //Make sure a hidden cache folder exists to store partial downloads
370 private static void CreateHiddenFolder(string rootPath, string folderName)
372 if (String.IsNullOrWhiteSpace(rootPath))
373 throw new ArgumentNullException("rootPath");
374 if (!Path.IsPathRooted(rootPath))
375 throw new ArgumentException("rootPath");
376 if (String.IsNullOrWhiteSpace(folderName))
377 throw new ArgumentNullException("folderName");
378 Contract.EndContractBlock();
380 var folder = Path.Combine(rootPath, folderName);
381 if (!Directory.Exists(folder))
383 var info = Directory.CreateDirectory(folder);
384 info.Attributes |= FileAttributes.Hidden;
386 Log.InfoFormat("Created cache Folder: {0}", folder);
390 var info = new DirectoryInfo(folder);
391 if ((info.Attributes & FileAttributes.Hidden) == 0)
393 info.Attributes |= FileAttributes.Hidden;
394 Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
402 private void StartWatcherAgent()
404 if (Log.IsDebugEnabled)
405 Log.DebugFormat("Start Folder Monitoring [{0}]",RootPath);
407 AgentLocator<FileAgent>.Register(FileAgent,RootPath);
409 FileAgent.IdleTimeout = Settings.FileIdleTimeout;
410 FileAgent.StatusKeeper = StatusKeeper;
411 FileAgent.StatusNotification = StatusNotification;
412 FileAgent.Workflow = Workflow;
413 FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
414 FileAgent.Start(_accountInfo, RootPath);
420 AgentLocator<FileAgent>.Remove(RootPath);
434 public void Dispose()
437 GC.SuppressFinalize(this);
440 protected virtual void Dispose(bool disposing)
449 public void MoveFileStates(string oldPath, string newPath)
451 if (String.IsNullOrWhiteSpace(oldPath))
452 throw new ArgumentNullException("oldPath");
453 if (!Path.IsPathRooted(oldPath))
454 throw new ArgumentException("oldPath must be an absolute path","oldPath");
455 if (string.IsNullOrWhiteSpace(newPath))
456 throw new ArgumentNullException("newPath");
457 if (!Path.IsPathRooted(newPath))
458 throw new ArgumentException("newPath must be an absolute path","newPath");
459 Contract.EndContractBlock();
461 StatusKeeper.ChangeRoots(oldPath, newPath);
464 public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed)
466 //Convert the uris to paths
467 var selectivePaths = UrisToFilePaths(uris);
469 var selectiveUri = uris.ToList();
470 this.Selectives.SetSelectedUris(_accountInfo,selectiveUri);
472 var removedPaths = UrisToFilePaths(removed);
473 UnversionSelectivePaths(removedPaths);
478 /// Mark all unselected paths as Unversioned
480 /// <param name="removed"></param>
481 private void UnversionSelectivePaths(List<string> removed)
486 //Ensure we remove any file state below the deleted folders
487 FileState.UnversionPaths(removed);
492 /// Return a list of absolute filepaths from a list of Uris
494 /// <param name="uris"></param>
495 /// <returns></returns>
496 private List<string> UrisToFilePaths(IEnumerable<Uri> uris)
499 return new List<string>();
501 var own = (from uri in uris
502 where uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())
503 let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()
504 //Trim the account name
505 select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList();
506 var others= (from uri in uris
507 where !uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())
508 let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()
509 //Trim the account name
510 select Path.Combine(RootPath,"others-shared", relativePath)).ToList();
511 return own.Union(others).ToList();
515 public ObjectInfo GetObjectInfo(string filePath)
517 if (String.IsNullOrWhiteSpace(filePath))
518 throw new ArgumentNullException("filePath");
519 Contract.EndContractBlock();
521 var file=new FileInfo(filePath);
522 string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
523 var relativePath = file.AsRelativeTo(RootPath);
525 string accountName,container;
527 var parts=relativePath.Split('\\');
529 var accountInfo = _accountInfo;
530 if (relativePath.StartsWith(FolderConstants.OthersFolder))
532 accountName = parts[1];
533 container = parts[2];
534 relativeUrl = String.Join("/", parts.Splice(3));
535 //Create the root URL for the target account
536 var oldName = UserName;
537 var absoluteUri = _accountInfo.StorageUri.AbsoluteUri;
538 var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
539 var root=absoluteUri.Substring(0, nameIndex);
541 accountInfo = new AccountInfo
543 UserName = accountName,
544 AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
545 StorageUri = new Uri(root + accountName),
546 BlockHash=accountInfo.BlockHash,
547 BlockSize=accountInfo.BlockSize,
548 Token=accountInfo.Token
553 accountName = UserName;
554 container = parts[0];
555 relativeUrl = String.Join("/", parts.Splice(1));
558 var client = new CloudFilesClient(accountInfo);
559 var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
563 public Task<ContainerInfo> GetContainerInfo(string filePath)
565 if (String.IsNullOrWhiteSpace(filePath))
566 throw new ArgumentNullException("filePath");
567 Contract.EndContractBlock();
569 var file=new FileInfo(filePath);
570 var relativePath = file.AsRelativeTo(RootPath);
572 string accountName,container;
574 var parts=relativePath.Split('\\');
576 var accountInfo = _accountInfo;
577 if (relativePath.StartsWith(FolderConstants.OthersFolder))
579 accountName = parts[1];
580 container = parts[2];
581 //Create the root URL for the target account
582 var oldName = UserName;
583 var absoluteUri = _accountInfo.StorageUri.AbsoluteUri;
584 var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
585 var root=absoluteUri.Substring(0, nameIndex);
587 accountInfo = new AccountInfo
589 UserName = accountName,
590 AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
591 StorageUri = new Uri(root + accountName),
592 BlockHash=accountInfo.BlockHash,
593 BlockSize=accountInfo.BlockSize,
594 Token=accountInfo.Token
599 accountName = UserName;
600 container = parts[0];
603 return Task.Factory.StartNew(() =>
605 var client = new CloudFilesClient(accountInfo);
606 var containerInfo = client.GetContainerInfo(accountName, container);
607 return containerInfo;