Disabled batch filtering
[pithos-ms-client] / trunk / Pithos.Core / PithosMonitor.cs
1 #region\r
2 /* -----------------------------------------------------------------------\r
3  * <copyright file="PithosMonitor.cs" company="GRNet">\r
4  * \r
5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
6  *\r
7  * Redistribution and use in source and binary forms, with or\r
8  * without modification, are permitted provided that the following\r
9  * conditions are met:\r
10  *\r
11  *   1. Redistributions of source code must retain the above\r
12  *      copyright notice, this list of conditions and the following\r
13  *      disclaimer.\r
14  *\r
15  *   2. Redistributions in binary form must reproduce the above\r
16  *      copyright notice, this list of conditions and the following\r
17  *      disclaimer in the documentation and/or other materials\r
18  *      provided with the distribution.\r
19  *\r
20  *\r
21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
32  * POSSIBILITY OF SUCH DAMAGE.\r
33  *\r
34  * The views and conclusions contained in the software and\r
35  * documentation are those of the authors and should not be\r
36  * interpreted as representing official policies, either expressed\r
37  * or implied, of GRNET S.A.\r
38  * </copyright>\r
39  * -----------------------------------------------------------------------\r
40  */\r
41 #endregion\r
42 using System;\r
43 using System.Collections.Generic;\r
44 using System.ComponentModel.Composition;\r
45 using System.Diagnostics.Contracts;\r
46 using System.IO;\r
47 using System.Linq;\r
48 using System.Reflection;\r
49 using System.Threading;\r
50 using System.Threading.Tasks;\r
51 using Pithos.Core.Agents;\r
52 using Pithos.Interfaces;\r
53 using Pithos.Network;\r
54 using log4net;\r
55 \r
56 namespace Pithos.Core\r
57 {\r
58     [Export(typeof(PithosMonitor))]\r
59     public class PithosMonitor:IDisposable\r
60     {\r
61         private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
62 \r
63         private int _blockSize;\r
64         private string _blockHash;\r
65 \r
66         [Import]\r
67         public IPithosSettings Settings{get;set;}\r
68 \r
69         private IStatusKeeper _statusKeeper;\r
70 \r
71         [Import]\r
72         public IStatusKeeper StatusKeeper\r
73         {\r
74             get { return _statusKeeper; }\r
75             set\r
76             {\r
77                 _statusKeeper = value;\r
78                 FileAgent.StatusKeeper = value;\r
79             }\r
80         }\r
81 \r
82 \r
83 \r
84 \r
85         private IPithosWorkflow _workflow;\r
86 \r
87         [Import]\r
88         public IPithosWorkflow Workflow\r
89         {\r
90             get { return _workflow; }\r
91             set\r
92             {\r
93                 _workflow = value;\r
94                 FileAgent.Workflow = value;\r
95             }\r
96         }\r
97 \r
98         public ICloudClient CloudClient { get; set; }\r
99 \r
100         public IStatusNotification StatusNotification { get; set; }\r
101 \r
102         //[Import]\r
103         public FileAgent FileAgent { get; private set; }\r
104 \r
105 /*\r
106         private WorkflowAgent _workflowAgent;\r
107 \r
108         [Import]\r
109         public WorkflowAgent WorkflowAgent\r
110         {\r
111             get { return _workflowAgent; }\r
112             set\r
113             {\r
114                 _workflowAgent = value;\r
115                 //FileAgent.WorkflowAgent = value;\r
116             }\r
117         }\r
118 */\r
119         \r
120         [Import]\r
121         public NetworkAgent NetworkAgent { get; set; }\r
122 \r
123         private PollAgent _pollAgent;\r
124 \r
125         [Import]\r
126         public PollAgent PollAgent\r
127         {\r
128             get { return _pollAgent; }\r
129             set\r
130             {\r
131                 _pollAgent = value;\r
132                 FileAgent.PollAgent = value;\r
133             }\r
134         }\r
135 \r
136         private Selectives _selectives;\r
137 \r
138         [Import]\r
139         public Selectives Selectives\r
140         {\r
141             get { return _selectives; }\r
142             set\r
143             {\r
144                 _selectives = value;\r
145                 FileAgent.Selectives = value;\r
146             }\r
147         }\r
148 \r
149         public string UserName { get; set; }\r
150         private string _apiKey;\r
151         public string ApiKey\r
152         {\r
153             get { return _apiKey; }\r
154             set\r
155             {\r
156                 _apiKey = value;\r
157                 if (_accountInfo != null)\r
158                     _accountInfo.Token = value;\r
159             }\r
160         }\r
161 \r
162         private AccountInfo _accountInfo;\r
163 \r
164         public AccountInfo Account\r
165         {\r
166             get { return _accountInfo; }\r
167         }\r
168 \r
169 \r
170 \r
171 \r
172 \r
173         public bool Pause { get; set; }       \r
174         /*public bool Pause\r
175         {\r
176             get { return FileAgent.Pause; }\r
177             set\r
178             {\r
179                 FileAgent.Pause = value;\r
180             }\r
181         }*/\r
182 \r
183         private string _rootPath;\r
184         public string RootPath\r
185         {\r
186             get { return _rootPath; }\r
187             set \r
188             {\r
189                 _rootPath = String.IsNullOrWhiteSpace(value) \r
190                     ? String.Empty \r
191                     : value.ToLower();\r
192             }\r
193         }\r
194 \r
195 \r
196         CancellationTokenSource _cancellationSource;\r
197 \r
198         public PithosMonitor()\r
199         {\r
200             FileAgent = new FileAgent();            \r
201         }\r
202         private bool _started;\r
203 \r
204         public void Start()\r
205         {            \r
206             if (String.IsNullOrWhiteSpace(ApiKey))\r
207                 throw new InvalidOperationException("The ApiKey is empty");\r
208             if (String.IsNullOrWhiteSpace(UserName))\r
209                 throw new InvalidOperationException("The UserName is empty");\r
210             if (String.IsNullOrWhiteSpace(AuthenticationUrl))\r
211                 throw new InvalidOperationException("The Authentication url is empty");\r
212             Contract.EndContractBlock();\r
213 \r
214             //If the account doesn't have a valid path, don't start monitoring but don't throw either\r
215             if (String.IsNullOrWhiteSpace(RootPath))\r
216                 //TODO; Warn user?\r
217                 return;\r
218 \r
219             //WorkflowAgent.StatusNotification = StatusNotification;\r
220 \r
221             StatusNotification.NotifyChange("Starting");\r
222             if (_started)\r
223             {\r
224                 if (!_cancellationSource.IsCancellationRequested)\r
225                     return;\r
226             }\r
227             _cancellationSource = new CancellationTokenSource();\r
228 \r
229             lock (this)\r
230             {\r
231                 CloudClient = new CloudFilesClient(UserName, ApiKey)\r
232                                   {UsePithos = true, AuthenticationUrl = AuthenticationUrl};\r
233                 _accountInfo = CloudClient.Authenticate();\r
234             }\r
235             _accountInfo.SiteUri = AuthenticationUrl;\r
236             _accountInfo.AccountPath = RootPath;\r
237 \r
238 \r
239             var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);\r
240             if (!Directory.Exists(pithosFolder))\r
241                 Directory.CreateDirectory(pithosFolder);\r
242             //Create the cache folder and ensure it is hidden\r
243             CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);\r
244 \r
245             var policy=CloudClient.GetAccountPolicies(_accountInfo);\r
246 \r
247             StatusNotification.NotifyAccount(policy);\r
248             EnsurePithosContainers();\r
249             \r
250             StatusKeeper.BlockHash = _blockHash;\r
251             StatusKeeper.BlockSize = _blockSize;\r
252             \r
253             StatusKeeper.StartProcessing(_cancellationSource.Token);\r
254             CleanupUnselectedStates();\r
255             IndexLocalFiles();\r
256             //Extract the URIs from the string collection\r
257             var settings = Settings.Accounts.First(s => s.AccountKey == _accountInfo.AccountKey );\r
258                             \r
259             var selectiveUrls=settings.SelectiveFolders.Cast<string>().Select(url => new Uri(url,UriKind.RelativeOrAbsolute))\r
260                 .Where(uri=>uri.IsAbsoluteUri).ToArray();\r
261 \r
262             SetSelectivePaths(selectiveUrls,null,null);\r
263             \r
264             StartWatcherAgent();\r
265 \r
266             StartNetworkAgent();\r
267             \r
268             //WorkflowAgent.RestartInterruptedFiles(_accountInfo);\r
269             _started = true;\r
270         }\r
271 \r
272         private void EnsurePithosContainers()\r
273         {\r
274 \r
275             //Create the two default containers if they are missing\r
276             var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};\r
277             foreach (var container in pithosContainers)\r
278             {                \r
279                 var info=CloudClient.GetContainerInfo(UserName, container);\r
280                 if (info == ContainerInfo.Empty)\r
281                 {\r
282                     CloudClient.CreateContainer(UserName, container);\r
283                     info = CloudClient.GetContainerInfo(UserName, container);\r
284                 }\r
285                 _blockSize = info.BlockSize;\r
286                 _blockHash = info.BlockHash;\r
287                 _accountInfo.BlockSize = _blockSize;\r
288                 _accountInfo.BlockHash = _blockHash;\r
289             }\r
290         }\r
291 \r
292         public string AuthenticationUrl { get; set; }\r
293 \r
294         private void IndexLocalFiles()\r
295         {\r
296             using (ThreadContext.Stacks["Operation"].Push("Indexing local files"))\r
297             {\r
298                 \r
299                 try\r
300                 {\r
301                     //StatusNotification.NotifyChange("Indexing Local Files");\r
302                     Log.Info("Start local indexing");\r
303                     StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Indexing Local Files");                    \r
304 \r
305                     var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);\r
306                     var directory = new DirectoryInfo(RootPath);\r
307                     var files =\r
308                         from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)\r
309                         where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&\r
310                               !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)\r
311                         select file;\r
312                     StatusKeeper.ProcessExistingFiles(files);\r
313 \r
314                 }\r
315                 catch (Exception exc)\r
316                 {\r
317                     Log.Error("[ERROR]", exc);\r
318                 }\r
319                 finally\r
320                 {\r
321                     Log.Info("[END]");\r
322                 }\r
323                 StatusNotification.SetPithosStatus(PithosStatus.LocalComplete,"Indexing Completed");\r
324             }\r
325         }\r
326 \r
327         \r
328   \r
329 \r
330 \r
331        /* private void StartWorkflowAgent()\r
332         {\r
333             WorkflowAgent.StatusNotification = StatusNotification;\r
334 \r
335 /*            //On Vista and up we can check for a network connection\r
336             bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;\r
337             //If we are not connected retry later\r
338             if (!connected)\r
339             {\r
340                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);\r
341                 return;\r
342             }#1#\r
343 \r
344             try\r
345             {\r
346                 WorkflowAgent.Start();                \r
347             }\r
348             catch (Exception)\r
349             {\r
350                 //Faild to authenticate due to network or account error\r
351                 //Retry after a while\r
352                 Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);\r
353             }\r
354         }*/\r
355 \r
356 \r
357         private void StartNetworkAgent()\r
358         {\r
359             NetworkAgent.StatusNotification = StatusNotification;\r
360 \r
361             //TODO: The Network and Poll agents are not account specific\r
362             //They should be moved outside PithosMonitor\r
363 /*\r
364             NetworkAgent.Start();\r
365 */\r
366 \r
367             PollAgent.AddAccount(_accountInfo);\r
368 \r
369             PollAgent.StatusNotification = StatusNotification;\r
370 \r
371             PollAgent.PollRemoteFiles();\r
372         }\r
373 \r
374         //Make sure a hidden cache folder exists to store partial downloads\r
375         private static void CreateHiddenFolder(string rootPath, string folderName)\r
376         {\r
377             if (String.IsNullOrWhiteSpace(rootPath))\r
378                 throw new ArgumentNullException("rootPath");\r
379             if (!Path.IsPathRooted(rootPath))\r
380                 throw new ArgumentException("rootPath");\r
381             if (String.IsNullOrWhiteSpace(folderName))\r
382                 throw new ArgumentNullException("folderName");\r
383             Contract.EndContractBlock();\r
384 \r
385             var folder = Path.Combine(rootPath, folderName);\r
386             if (!Directory.Exists(folder))\r
387             {\r
388                 var info = Directory.CreateDirectory(folder);\r
389                 info.Attributes |= FileAttributes.Hidden;\r
390 \r
391                 Log.InfoFormat("Created cache Folder: {0}", folder);\r
392             }\r
393             else\r
394             {\r
395                 var info = new DirectoryInfo(folder);\r
396                 if ((info.Attributes & FileAttributes.Hidden) == 0)\r
397                 {\r
398                     info.Attributes |= FileAttributes.Hidden;\r
399                     Log.InfoFormat("Reset cache folder to hidden: {0}", folder);\r
400                 }                                \r
401             }\r
402         }\r
403 \r
404        \r
405 \r
406 \r
407         private void StartWatcherAgent()\r
408         {\r
409             if (Log.IsDebugEnabled)\r
410                 Log.DebugFormat("Start Folder Monitoring [{0}]",RootPath);\r
411 \r
412             AgentLocator<FileAgent>.Register(FileAgent,RootPath);\r
413             \r
414             FileAgent.IdleTimeout = Settings.FileIdleTimeout;\r
415             FileAgent.StatusKeeper = StatusKeeper;\r
416             FileAgent.StatusNotification = StatusNotification;\r
417             FileAgent.Workflow = Workflow;\r
418             FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);\r
419             FileAgent.Start(_accountInfo, RootPath);\r
420         }\r
421 \r
422         public void Stop()\r
423         {\r
424 /*\r
425             AgentLocator<FileAgent>.Remove(RootPath);\r
426 \r
427             if (FileAgent!=null)\r
428                 FileAgent.Stop();\r
429             FileAgent = null;\r
430 */\r
431         }\r
432 \r
433 \r
434         ~PithosMonitor()\r
435         {\r
436             Dispose(false);\r
437         }\r
438 \r
439         public void Dispose()\r
440         {\r
441             Dispose(true);\r
442             GC.SuppressFinalize(this);\r
443         }\r
444 \r
445         protected virtual void Dispose(bool disposing)\r
446         {\r
447             if (disposing)\r
448             {\r
449                 Stop();\r
450             }\r
451         }\r
452 \r
453 \r
454         public void MoveFileStates(string oldPath, string newPath)\r
455         {\r
456             if (String.IsNullOrWhiteSpace(oldPath))\r
457                 throw new ArgumentNullException("oldPath");\r
458             if (!Path.IsPathRooted(oldPath))\r
459                 throw new ArgumentException("oldPath must be an absolute path","oldPath");\r
460             if (string.IsNullOrWhiteSpace(newPath))\r
461                 throw new ArgumentNullException("newPath");\r
462             if (!Path.IsPathRooted(newPath))\r
463                 throw new ArgumentException("newPath must be an absolute path","newPath");\r
464             Contract.EndContractBlock();\r
465 \r
466             StatusKeeper.ChangeRoots(oldPath, newPath);\r
467         }\r
468 \r
469         private void CleanupUnselectedStates()\r
470         {\r
471             //var settings = Settings.Accounts.First(s => s.AccountKey == _accountInfo.AccountKey);\r
472             if (!Selectives.IsSelectiveEnabled(_accountInfo.AccountKey)) return;\r
473 \r
474             List<string> selectivePaths;\r
475             if (Selectives.SelectivePaths.TryGetValue(_accountInfo.AccountKey, out selectivePaths))\r
476             {\r
477                 var statePaths= FileState.Queryable.Select(state => state.FilePath).ToList();\r
478                 var removedPaths = statePaths.Where(sp => !selectivePaths.Any(sp.IsAtOrBelow));\r
479 \r
480                 UnversionSelectivePaths(removedPaths.ToList());\r
481             }\r
482             else\r
483             {\r
484                 StatusKeeper.ClearFolderStatus(Account.AccountPath);\r
485             }\r
486         }\r
487 \r
488         public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed)\r
489         {\r
490             //Convert the uris to paths\r
491             //var selectivePaths = UrisToFilePaths(uris);\r
492             \r
493             var selectiveUri = uris.ToList();\r
494             this.Selectives.SetSelectedUris(_accountInfo,selectiveUri);\r
495 \r
496             var removedPaths = UrisToFilePaths(removed);\r
497             UnversionSelectivePaths(removedPaths);\r
498 \r
499         }\r
500 \r
501         /// <summary>\r
502         /// Mark all unselected paths as Unversioned\r
503         /// </summary>\r
504         /// <param name="removed"></param>\r
505         private void UnversionSelectivePaths(List<string> removed)\r
506         {\r
507             if (removed == null)\r
508                 return;\r
509 \r
510             //Ensure we remove any file state below the deleted folders\r
511             FileState.UnversionPaths(removed);\r
512         }\r
513 \r
514 \r
515         /// <summary>\r
516         /// Return a list of absolute filepaths from a list of Uris\r
517         /// </summary>\r
518         /// <param name="uris"></param>\r
519         /// <returns></returns>\r
520         public List<string> UrisToFilePaths(IEnumerable<Uri> uris)\r
521         {\r
522             if (uris == null)\r
523                 return new List<string>();\r
524 \r
525             var own = (from uri in uris\r
526                        where uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())\r
527                                    let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()\r
528                                    //Trim the account name\r
529                                    select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList();\r
530             var others= (from uri in uris\r
531                          where !uri.ToString().StartsWith(_accountInfo.StorageUri.ToString())\r
532                                    let relativePath = _accountInfo.StorageUri.MakeRelativeUri(uri).RelativeUriToFilePath()\r
533                                    //Trim the account name\r
534                                    select Path.Combine(RootPath,"others-shared", relativePath)).ToList();\r
535             return own.Union(others).ToList();            \r
536         }\r
537 \r
538 \r
539         public ObjectInfo GetObjectInfo(string filePath)\r
540         {\r
541             if (String.IsNullOrWhiteSpace(filePath))\r
542                 throw new ArgumentNullException("filePath");\r
543             Contract.EndContractBlock();\r
544 \r
545             var file=new FileInfo(filePath);\r
546             string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);\r
547             var relativePath = file.AsRelativeTo(RootPath);\r
548             \r
549             string accountName,container;\r
550             \r
551             var parts=relativePath.Split('\\');\r
552 \r
553             var accountInfo = _accountInfo;\r
554             if (relativePath.StartsWith(FolderConstants.OthersFolder))\r
555             {                \r
556                 accountName = parts[1];\r
557                 container = parts[2];\r
558                 relativeUrl = String.Join("/", parts.Splice(3));\r
559                 //Create the root URL for the target account\r
560                 var oldName = UserName;\r
561                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;\r
562                 var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);\r
563                 var root=absoluteUri.Substring(0, nameIndex);\r
564 \r
565                 accountInfo = new AccountInfo\r
566                 {\r
567                     UserName = accountName,\r
568                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),\r
569                     StorageUri = new Uri(root + accountName),\r
570                     BlockHash=accountInfo.BlockHash,\r
571                     BlockSize=accountInfo.BlockSize,\r
572                     Token=accountInfo.Token\r
573                 };\r
574             }\r
575             else\r
576             {\r
577                 accountName = UserName;\r
578                 container = parts[0];\r
579                 relativeUrl = String.Join("/", parts.Splice(1));\r
580             }\r
581             \r
582             var client = new CloudFilesClient(accountInfo);\r
583             var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);\r
584             return objectInfo;\r
585         }\r
586         \r
587         public Task<ContainerInfo> GetContainerInfo(string filePath)\r
588         {\r
589             if (String.IsNullOrWhiteSpace(filePath))\r
590                 throw new ArgumentNullException("filePath");\r
591             Contract.EndContractBlock();\r
592 \r
593             var file=new FileInfo(filePath);\r
594             var relativePath = file.AsRelativeTo(RootPath);\r
595             \r
596             string accountName,container;\r
597             \r
598             var parts=relativePath.Split('\\');\r
599 \r
600             var accountInfo = _accountInfo;\r
601             if (relativePath.StartsWith(FolderConstants.OthersFolder))\r
602             {                \r
603                 accountName = parts[1];\r
604                 container = parts[2];                \r
605                 //Create the root URL for the target account\r
606                 var oldName = UserName;\r
607                 var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;\r
608                 var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);\r
609                 var root=absoluteUri.Substring(0, nameIndex);\r
610 \r
611                 accountInfo = new AccountInfo\r
612                 {\r
613                     UserName = accountName,\r
614                     AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),\r
615                     StorageUri = new Uri(root + accountName),\r
616                     BlockHash=accountInfo.BlockHash,\r
617                     BlockSize=accountInfo.BlockSize,\r
618                     Token=accountInfo.Token\r
619                 };\r
620             }\r
621             else\r
622             {\r
623                 accountName = UserName;\r
624                 container = parts[0];                \r
625             }\r
626 \r
627             return Task.Factory.StartNew(() =>\r
628             {\r
629                 var client = new CloudFilesClient(accountInfo);\r
630                 var containerInfo = client.GetContainerInfo(accountName, container);\r
631                 return containerInfo;\r
632             });\r
633         }\r
634     }\r
635 }\r