2 /* -----------------------------------------------------------------------
\r
3 * <copyright file="PithosMonitor.cs" company="GRNet">
\r
5 * Copyright 2011-2012 GRNET S.A. All rights reserved.
\r
7 * Redistribution and use in source and binary forms, with or
\r
8 * without modification, are permitted provided that the following
\r
9 * conditions are met:
\r
11 * 1. Redistributions of source code must retain the above
\r
12 * copyright notice, this list of conditions and the following
\r
15 * 2. Redistributions in binary form must reproduce the above
\r
16 * copyright notice, this list of conditions and the following
\r
17 * disclaimer in the documentation and/or other materials
\r
18 * provided with the distribution.
\r
21 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
\r
22 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
\r
23 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
\r
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
\r
25 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
\r
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
\r
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
\r
28 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
\r
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
\r
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
\r
31 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
\r
32 * POSSIBILITY OF SUCH DAMAGE.
\r
34 * The views and conclusions contained in the software and
\r
35 * documentation are those of the authors and should not be
\r
36 * interpreted as representing official policies, either expressed
\r
37 * or implied, of GRNET S.A.
\r
39 * -----------------------------------------------------------------------
\r
43 using System.Collections.Generic;
\r
44 using System.ComponentModel.Composition;
\r
45 using System.Diagnostics.Contracts;
\r
48 using System.Reflection;
\r
49 using System.Threading;
\r
50 using System.Threading.Tasks;
\r
51 using Pithos.Core.Agents;
\r
52 using Pithos.Interfaces;
\r
53 using Pithos.Network;
\r
56 namespace Pithos.Core
\r
58 [Export(typeof(PithosMonitor))]
\r
59 public class PithosMonitor:IDisposable
\r
61 private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
\r
63 private int _blockSize;
\r
64 private string _blockHash;
\r
67 public IPithosSettings Settings{get;set;}
\r
69 private IStatusKeeper _statusKeeper;
\r
72 public IStatusKeeper StatusKeeper
\r
74 get { return _statusKeeper; }
\r
77 _statusKeeper = value;
\r
78 FileAgent.StatusKeeper = value;
\r
85 private IPithosWorkflow _workflow;
\r
88 public IPithosWorkflow Workflow
\r
90 get { return _workflow; }
\r
94 FileAgent.Workflow = value;
\r
98 public ICloudClient CloudClient { get; set; }
\r
100 public IStatusNotification StatusNotification { get; set; }
\r
103 public FileAgent FileAgent { get; private set; }
\r
106 private WorkflowAgent _workflowAgent;
\r
109 public WorkflowAgent WorkflowAgent
\r
111 get { return _workflowAgent; }
\r
114 _workflowAgent = value;
\r
115 //FileAgent.WorkflowAgent = value;
\r
121 public NetworkAgent NetworkAgent { get; set; }
\r
123 private PollAgent _pollAgent;
\r
126 public PollAgent PollAgent
\r
128 get { return _pollAgent; }
\r
131 _pollAgent = value;
\r
132 FileAgent.PollAgent = value;
\r
136 private Selectives _selectives;
\r
139 public Selectives Selectives
\r
141 get { return _selectives; }
\r
144 _selectives = value;
\r
145 FileAgent.Selectives = value;
\r
149 public string UserName { get; set; }
\r
150 private string _apiKey;
\r
151 public string ApiKey
\r
153 get { return _apiKey; }
\r
157 if (_accountInfo != null)
\r
158 _accountInfo.Token = value;
\r
162 private AccountInfo _accountInfo;
\r
164 public AccountInfo Account
\r
166 get { return _accountInfo; }
\r
173 public bool Pause { get; set; }
\r
174 /*public bool Pause
\r
176 get { return FileAgent.Pause; }
\r
179 FileAgent.Pause = value;
\r
183 private string _rootPath;
\r
184 public string RootPath
\r
186 get { return _rootPath; }
\r
189 _rootPath = String.IsNullOrWhiteSpace(value)
\r
196 CancellationTokenSource _cancellationSource;
\r
198 public PithosMonitor()
\r
200 FileAgent = new FileAgent();
\r
202 private bool _started;
\r
204 public void Start()
\r
206 if (String.IsNullOrWhiteSpace(ApiKey))
\r
207 throw new InvalidOperationException("The ApiKey is empty");
\r
208 if (String.IsNullOrWhiteSpace(UserName))
\r
209 throw new InvalidOperationException("The UserName is empty");
\r
210 if (String.IsNullOrWhiteSpace(AuthenticationUrl))
\r
211 throw new InvalidOperationException("The Authentication url is empty");
\r
212 Contract.EndContractBlock();
\r
214 //If the account doesn't have a valid path, don't start monitoring but don't throw either
\r
215 if (String.IsNullOrWhiteSpace(RootPath))
\r
219 //WorkflowAgent.StatusNotification = StatusNotification;
\r
221 StatusNotification.NotifyChange("Starting");
\r
224 if (!_cancellationSource.IsCancellationRequested)
\r
227 _cancellationSource = new CancellationTokenSource();
\r
231 CloudClient = new CloudFilesClient(UserName, ApiKey)
\r
232 {UsePithos = true, AuthenticationUrl = AuthenticationUrl};
\r
233 _accountInfo = CloudClient.Authenticate();
\r
235 _accountInfo.SiteUri = AuthenticationUrl;
\r
236 _accountInfo.AccountPath = RootPath;
\r
239 var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
\r
240 if (!Directory.Exists(pithosFolder))
\r
241 Directory.CreateDirectory(pithosFolder);
\r
242 //Create the cache folder and ensure it is hidden
\r
243 CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
\r
245 var policy=CloudClient.GetAccountPolicies(_accountInfo);
\r
247 StatusNotification.NotifyAccount(policy);
\r
248 EnsurePithosContainers();
\r
250 StatusKeeper.BlockHash = _blockHash;
\r
251 StatusKeeper.BlockSize = _blockSize;
\r
253 StatusKeeper.StartProcessing(_cancellationSource.Token);
\r
254 CleanupUnselectedStates();
\r
256 //Extract the URIs from the string collection
\r
257 var settings = Settings.Accounts.First(s => s.AccountKey == _accountInfo.AccountKey );
\r
259 var selectiveUrls=settings.SelectiveFolders.Cast<string>().Select(url => new Uri(url,UriKind.RelativeOrAbsolute))
\r
260 .Where(uri=>uri.IsAbsoluteUri).ToArray();
\r
262 SetSelectivePaths(selectiveUrls,null,null);
\r
264 StartWatcherAgent();
\r
266 StartNetworkAgent();
\r
268 //WorkflowAgent.RestartInterruptedFiles(_accountInfo);
\r
272 private void EnsurePithosContainers()
\r
275 //Create the two default containers if they are missing
\r
276 var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
\r
277 foreach (var container in pithosContainers)
\r
279 var info=CloudClient.GetContainerInfo(UserName, container);
\r
280 if (info == ContainerInfo.Empty)
\r
282 CloudClient.CreateContainer(UserName, container);
\r
283 info = CloudClient.GetContainerInfo(UserName, container);
\r
285 _blockSize = info.BlockSize;
\r
286 _blockHash = info.BlockHash;
\r
287 _accountInfo.BlockSize = _blockSize;
\r
288 _accountInfo.BlockHash = _blockHash;
\r
292 public string AuthenticationUrl { get; set; }
\r
294 private void IndexLocalFiles()
\r
296 using (ThreadContext.Stacks["Operation"].Push("Indexing local files"))
\r
301 //StatusNotification.NotifyChange("Indexing Local Files");
\r
302 Log.Info("Start local indexing");
\r
303 StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Indexing Local Files");
\r
305 var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
\r
306 var directory = new DirectoryInfo(RootPath);
\r
308 from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
\r
309 where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
\r
310 !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
\r
312 StatusKeeper.ProcessExistingFiles(files);
\r
315 catch (Exception exc)
\r
317 Log.Error("[ERROR]", exc);
\r
323 StatusNotification.SetPithosStatus(PithosStatus.LocalComplete,"Indexing Completed");
\r
331 /* private void StartWorkflowAgent()
\r
333 WorkflowAgent.StatusNotification = StatusNotification;
\r
335 /* //On Vista and up we can check for a network connection
\r
336 bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
\r
337 //If we are not connected retry later
\r
340 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
\r
346 WorkflowAgent.Start();
\r
350 //Faild to authenticate due to network or account error
\r
351 //Retry after a while
\r
352 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
\r
357 private void StartNetworkAgent()
\r
359 NetworkAgent.StatusNotification = StatusNotification;
\r
361 //TODO: The Network and Poll agents are not account specific
\r
362 //They should be moved outside PithosMonitor
\r
364 NetworkAgent.Start();
\r
367 PollAgent.AddAccount(_accountInfo);
\r
369 PollAgent.StatusNotification = StatusNotification;
\r
371 PollAgent.PollRemoteFiles();
\r
374 //Make sure a hidden cache folder exists to store partial downloads
\r
375 private static void CreateHiddenFolder(string rootPath, string folderName)
\r
377 if (String.IsNullOrWhiteSpace(rootPath))
\r
378 throw new ArgumentNullException("rootPath");
\r
379 if (!Path.IsPathRooted(rootPath))
\r
380 throw new ArgumentException("rootPath");
\r
381 if (String.IsNullOrWhiteSpace(folderName))
\r
382 throw new ArgumentNullException("folderName");
\r
383 Contract.EndContractBlock();
\r
385 var folder = Path.Combine(rootPath, folderName);
\r
386 if (!Directory.Exists(folder))
\r
388 var info = Directory.CreateDirectory(folder);
\r
389 info.Attributes |= FileAttributes.Hidden;
\r
391 Log.InfoFormat("Created cache Folder: {0}", folder);
\r
395 var info = new DirectoryInfo(folder);
\r
396 if ((info.Attributes & FileAttributes.Hidden) == 0)
\r
398 info.Attributes |= FileAttributes.Hidden;
\r
399 Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
\r
407 private void StartWatcherAgent()
\r
409 if (Log.IsDebugEnabled)
\r
410 Log.DebugFormat("Start Folder Monitoring [{0}]",RootPath);
\r
412 AgentLocator<FileAgent>.Register(FileAgent,RootPath);
\r
414 FileAgent.IdleTimeout = Settings.FileIdleTimeout;
\r
415 FileAgent.StatusKeeper = StatusKeeper;
\r
416 FileAgent.StatusNotification = StatusNotification;
\r
417 FileAgent.Workflow = Workflow;
\r
418 FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
\r
419 FileAgent.Start(_accountInfo, RootPath);
\r
425 AgentLocator<FileAgent>.Remove(RootPath);
\r
427 if (FileAgent!=null)
\r
439 public void Dispose()
\r
442 GC.SuppressFinalize(this);
\r
445 protected virtual void Dispose(bool disposing)
\r
454 public void MoveFileStates(string oldPath, string newPath)
\r
456 if (String.IsNullOrWhiteSpace(oldPath))
\r
457 throw new ArgumentNullException("oldPath");
\r
458 if (!Path.IsPathRooted(oldPath))
\r
459 throw new ArgumentException("oldPath must be an absolute path","oldPath");
\r
460 if (string.IsNullOrWhiteSpace(newPath))
\r
461 throw new ArgumentNullException("newPath");
\r
462 if (!Path.IsPathRooted(newPath))
\r
463 throw new ArgumentException("newPath must be an absolute path","newPath");
\r
464 Contract.EndContractBlock();
\r
466 StatusKeeper.ChangeRoots(oldPath, newPath);
\r
469 private void CleanupUnselectedStates()
\r
471 //var settings = Settings.Accounts.First(s => s.AccountKey == _accountInfo.AccountKey);
\r
472 if (!Selectives.IsSelectiveEnabled(_accountInfo.AccountKey)) return;
\r
474 List<string> selectivePaths;
\r
475 if (Selectives.SelectivePaths.TryGetValue(_accountInfo.AccountKey, out selectivePaths))
\r
477 var statePaths= FileState.Queryable.Select(state => state.FilePath).ToList();
\r
478 var removedPaths = statePaths.Where(sp => !selectivePaths.Any(sp.IsAtOrBelow));
\r
480 UnversionSelectivePaths(removedPaths.ToList());
\r
484 StatusKeeper.ClearFolderStatus(Account.AccountPath);
\r
488 public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed)
\r
490 //Convert the uris to paths
\r
491 //var selectivePaths = UrisToFilePaths(uris);
\r
493 var selectiveUri = uris.ToList();
\r
494 this.Selectives.SetSelectedUris(_accountInfo,selectiveUri);
\r
496 var removedPaths = UrisToFilePaths(removed);
\r
497 UnversionSelectivePaths(removedPaths);
\r
502 /// Mark all unselected paths as Unversioned
\r
504 /// <param name="removed"></param>
\r
505 private void UnversionSelectivePaths(List<string> removed)
\r
507 if (removed == null)
\r
510 //Ensure we remove any file state below the deleted folders
\r
511 FileState.UnversionPaths(removed);
\r
516 /// Return a list of absolute filepaths from a list of Uris
\r
518 /// <param name="uris"></param>
\r
519 /// <returns></returns>
\r
520 public List<string> UrisToFilePaths(IEnumerable<Uri> uris)
\r
523 return new List<string>();
\r
525 var own = (from uri in uris
\r
526 where uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())
\r
527 let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()
\r
528 //Trim the account name
\r
529 select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList();
\r
530 var others= (from uri in uris
\r
531 where !uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())
\r
532 let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()
\r
533 //Trim the account name
\r
534 select Path.Combine(RootPath,"others-shared", relativePath)).ToList();
\r
535 return own.Union(others).ToList();
\r
539 public ObjectInfo GetObjectInfo(string filePath)
\r
541 if (String.IsNullOrWhiteSpace(filePath))
\r
542 throw new ArgumentNullException("filePath");
\r
543 Contract.EndContractBlock();
\r
545 var file=new FileInfo(filePath);
\r
546 string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
\r
547 var relativePath = file.AsRelativeTo(RootPath);
\r
549 string accountName,container;
\r
551 var parts=relativePath.Split('\\');
\r
553 var accountInfo = _accountInfo;
\r
554 if (relativePath.StartsWith(FolderConstants.OthersFolder))
\r
556 accountName = parts[1];
\r
557 container = parts[2];
\r
558 relativeUrl = String.Join("/", parts.Splice(3));
\r
559 //Create the root URL for the target account
\r
560 var oldName = UserName;
\r
561 var absoluteUri = _accountInfo.StorageUri.AbsoluteUri;
\r
562 var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
\r
563 var root=absoluteUri.Substring(0, nameIndex);
\r
565 accountInfo = new AccountInfo
\r
567 UserName = accountName,
\r
568 AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
\r
569 StorageUri = new Uri(root + accountName),
\r
570 BlockHash=accountInfo.BlockHash,
\r
571 BlockSize=accountInfo.BlockSize,
\r
572 Token=accountInfo.Token
\r
577 accountName = UserName;
\r
578 container = parts[0];
\r
579 relativeUrl = String.Join("/", parts.Splice(1));
\r
582 var client = new CloudFilesClient(accountInfo);
\r
583 var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
\r
587 public Task<ContainerInfo> GetContainerInfo(string filePath)
\r
589 if (String.IsNullOrWhiteSpace(filePath))
\r
590 throw new ArgumentNullException("filePath");
\r
591 Contract.EndContractBlock();
\r
593 var file=new FileInfo(filePath);
\r
594 var relativePath = file.AsRelativeTo(RootPath);
\r
596 string accountName,container;
\r
598 var parts=relativePath.Split('\\');
\r
600 var accountInfo = _accountInfo;
\r
601 if (relativePath.StartsWith(FolderConstants.OthersFolder))
\r
603 accountName = parts[1];
\r
604 container = parts[2];
\r
605 //Create the root URL for the target account
\r
606 var oldName = UserName;
\r
607 var absoluteUri = _accountInfo.StorageUri.AbsoluteUri;
\r
608 var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
\r
609 var root=absoluteUri.Substring(0, nameIndex);
\r
611 accountInfo = new AccountInfo
\r
613 UserName = accountName,
\r
614 AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
\r
615 StorageUri = new Uri(root + accountName),
\r
616 BlockHash=accountInfo.BlockHash,
\r
617 BlockSize=accountInfo.BlockSize,
\r
618 Token=accountInfo.Token
\r
623 accountName = UserName;
\r
624 container = parts[0];
\r
627 return Task.Factory.StartNew(() =>
\r
629 var client = new CloudFilesClient(accountInfo);
\r
630 var containerInfo = client.GetContainerInfo(accountName, container);
\r
631 return containerInfo;
\r