Statistics
| Branch: | Revision:

root / trunk / Pithos.Core / PithosMonitor.cs @ dccd340f

History | View | Annotate | Download (19.1 kB)

1
#region
2
/* -----------------------------------------------------------------------
3
 * <copyright file="PithosMonitor.cs" company="GRNet">
4
 * 
5
 * Copyright 2011-2012 GRNET S.A. All rights reserved.
6
 *
7
 * Redistribution and use in source and binary forms, with or
8
 * without modification, are permitted provided that the following
9
 * conditions are met:
10
 *
11
 *   1. Redistributions of source code must retain the above
12
 *      copyright notice, this list of conditions and the following
13
 *      disclaimer.
14
 *
15
 *   2. Redistributions in binary form must reproduce the above
16
 *      copyright notice, this list of conditions and the following
17
 *      disclaimer in the documentation and/or other materials
18
 *      provided with the distribution.
19
 *
20
 *
21
 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32
 * POSSIBILITY OF SUCH DAMAGE.
33
 *
34
 * The views and conclusions contained in the software and
35
 * documentation are those of the authors and should not be
36
 * interpreted as representing official policies, either expressed
37
 * or implied, of GRNET S.A.
38
 * </copyright>
39
 * -----------------------------------------------------------------------
40
 */
41
#endregion
42
using System;
43
using System.Collections.Generic;
44
using System.ComponentModel.Composition;
45
using System.Diagnostics.Contracts;
46
using System.IO;
47
using System.Linq;
48
using System.Reflection;
49
using System.Threading;
50
using System.Threading.Tasks;
51
using Pithos.Core.Agents;
52
using Pithos.Interfaces;
53
using Pithos.Network;
54
using log4net;
55

    
56
namespace Pithos.Core
57
{
58
    [Export(typeof(PithosMonitor))]
59
    public class PithosMonitor:IDisposable
60
    {
61
        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
62

    
63
        private int _blockSize;
64
        private string _blockHash;
65

    
66
        [Import]
67
        public IPithosSettings Settings{get;set;}
68

    
69
        private IStatusKeeper _statusKeeper;
70

    
71
        [Import]
72
        public IStatusKeeper StatusKeeper
73
        {
74
            get { return _statusKeeper; }
75
            set
76
            {
77
                _statusKeeper = value;
78
                FileAgent.StatusKeeper = value;
79
            }
80
        }
81

    
82

    
83
        private IPithosWorkflow _workflow;
84

    
85
        [Import]
86
        public IPithosWorkflow Workflow
87
        {
88
            get { return _workflow; }
89
            set
90
            {
91
                _workflow = value;
92
                FileAgent.Workflow = value;
93
            }
94
        }
95

    
96
        public ICloudClient CloudClient { get; set; }
97

    
98
        public IStatusNotification StatusNotification { get; set; }
99

    
100
        //[Import]
101
        public FileAgent FileAgent { get; private set; }
102

    
103
        private WorkflowAgent _workflowAgent;
104

    
105
        [Import]
106
        public WorkflowAgent WorkflowAgent
107
        {
108
            get { return _workflowAgent; }
109
            set
110
            {
111
                _workflowAgent = value;
112
                FileAgent.WorkflowAgent = value;
113
            }
114
        }
115
        
116
        [Import]
117
        public NetworkAgent NetworkAgent { get; set; }
118
        [Import]
119
        public PollAgent PollAgent { get; set; }       
120

    
121
        public string UserName { get; set; }
122
        private string _apiKey;
123
        public string ApiKey
124
        {
125
            get { return _apiKey; }
126
            set
127
            {
128
                _apiKey = value;
129
                if (_accountInfo != null)
130
                    _accountInfo.Token = value;
131
            }
132
        }
133

    
134
        private AccountInfo _accountInfo;
135

    
136

    
137

    
138

    
139

    
140

    
141
        public bool Pause
142
        {
143
            get { return FileAgent.Pause; }
144
            set
145
            {
146
                FileAgent.Pause = value;
147
            }
148
        }
149

    
150
        private string _rootPath;
151
        public string RootPath
152
        {
153
            get { return _rootPath; }
154
            set 
155
            {
156
                _rootPath = String.IsNullOrWhiteSpace(value) 
157
                    ? String.Empty 
158
                    : value.ToLower();
159
            }
160
        }
161

    
162

    
163
        CancellationTokenSource _cancellationSource;
164

    
165
        public PithosMonitor()
166
        {
167
            FileAgent = new FileAgent();
168
        }
169
        private bool _started;
170

    
171
        public void Start()
172
        {            
173
            if (String.IsNullOrWhiteSpace(ApiKey))
174
                throw new InvalidOperationException("The ApiKey is empty");
175
            if (String.IsNullOrWhiteSpace(UserName))
176
                throw new InvalidOperationException("The UserName is empty");
177
            if (String.IsNullOrWhiteSpace(AuthenticationUrl))
178
                throw new InvalidOperationException("The Authentication url is empty");
179
            Contract.EndContractBlock();
180

    
181
            //If the account doesn't have a valid path, don't start monitoring but don't throw either
182
            if (String.IsNullOrWhiteSpace(RootPath))
183
                //TODO; Warn user?
184
                return;
185

    
186
            WorkflowAgent.StatusNotification = StatusNotification;
187

    
188
            StatusNotification.NotifyChange("Starting");
189
            if (_started)
190
            {
191
                if (!_cancellationSource.IsCancellationRequested)
192
                    return;
193
            }
194
            _cancellationSource = new CancellationTokenSource();
195

    
196

    
197
            CloudClient = new CloudFilesClient(UserName, ApiKey)
198
                              {UsePithos = true, AuthenticationUrl = AuthenticationUrl};
199

    
200

    
201
            _accountInfo = CloudClient.Authenticate();            
202
            _accountInfo.SiteUri = AuthenticationUrl;
203
            _accountInfo.AccountPath = RootPath;
204

    
205

    
206
            var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
207
            if (!Directory.Exists(pithosFolder))
208
                Directory.CreateDirectory(pithosFolder);
209
            //Create the cache folder and ensure it is hidden
210
            CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
211

    
212
            var policy=CloudClient.GetAccountPolicies(_accountInfo);
213

    
214
            StatusNotification.NotifyAccount(policy);
215
            EnsurePithosContainers();
216
            
217
            StatusKeeper.BlockHash = _blockHash;
218
            StatusKeeper.BlockSize = _blockSize;
219
            
220
            StatusKeeper.StartProcessing(_cancellationSource.Token);
221
            IndexLocalFiles();
222
            //Extract the URIs from the string collection
223
            var settings = Settings.Accounts.First(s => s.AccountName == _accountInfo.UserName);
224
            var selectiveUrls=settings.SelectiveFolders.Cast<string>().Select(url => new Uri(url)).ToArray();
225

    
226
            SetSelectivePaths(selectiveUrls,null,null);
227
            
228
            StartWatcherAgent();
229

    
230
            StartNetworkAgent();
231
            
232
            WorkflowAgent.RestartInterruptedFiles(_accountInfo);
233
            _started = true;
234
        }
235

    
236
        private void EnsurePithosContainers()
237
        {
238

    
239
            //Create the two default containers if they are missing
240
            var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
241
            foreach (var container in pithosContainers)
242
            {                
243
                var info=CloudClient.GetContainerInfo(UserName, container);
244
                if (info == ContainerInfo.Empty)
245
                {
246
                    CloudClient.CreateContainer(UserName, container);
247
                    info = CloudClient.GetContainerInfo(UserName, container);
248
                }
249
                _blockSize = info.BlockSize;
250
                _blockHash = info.BlockHash;
251
                _accountInfo.BlockSize = _blockSize;
252
                _accountInfo.BlockHash = _blockHash;
253
            }
254
        }
255

    
256
        public string AuthenticationUrl { get; set; }
257

    
258
        private void IndexLocalFiles()
259
        {
260
            using (ThreadContext.Stacks["Operation"].Push("Indexing local files"))
261
            {
262
                
263
                try
264
                {
265
                    //StatusNotification.NotifyChange("Indexing Local Files");
266
                    Log.Info("Start local indexing");
267
                    StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Indexing Local Files");                    
268

    
269
                    var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
270
                    var directory = new DirectoryInfo(RootPath);
271
                    var files =
272
                        from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
273
                        where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
274
                              !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
275
                        select file;
276
                    StatusKeeper.ProcessExistingFiles(files);
277

    
278
                }
279
                catch (Exception exc)
280
                {
281
                    Log.Error("[ERROR]", exc);
282
                }
283
                finally
284
                {
285
                    Log.Info("[END]");
286
                }
287
                StatusNotification.SetPithosStatus(PithosStatus.LocalComplete,"Indexing Completed");
288
            }
289
        }
290

    
291
        
292
  
293

    
294

    
295
       /* private void StartWorkflowAgent()
296
        {
297
            WorkflowAgent.StatusNotification = StatusNotification;
298

    
299
/*            //On Vista and up we can check for a network connection
300
            bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
301
            //If we are not connected retry later
302
            if (!connected)
303
            {
304
                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
305
                return;
306
            }#1#
307

    
308
            try
309
            {
310
                WorkflowAgent.Start();                
311
            }
312
            catch (Exception)
313
            {
314
                //Faild to authenticate due to network or account error
315
                //Retry after a while
316
                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
317
            }
318
        }*/
319

    
320

    
321
        private void StartNetworkAgent()
322
        {
323
            NetworkAgent.StatusNotification = StatusNotification;
324

    
325
            //TODO: The Network and Poll agents are not account specific
326
            //They should be moved outside PithosMonitor
327
            NetworkAgent.Start();
328

    
329
            PollAgent.AddAccount(_accountInfo);
330

    
331
            PollAgent.StatusNotification = StatusNotification;
332

    
333
            PollAgent.PollRemoteFiles();
334
        }
335

    
336
        //Make sure a hidden cache folder exists to store partial downloads
337
        private static void CreateHiddenFolder(string rootPath, string folderName)
338
        {
339
            if (String.IsNullOrWhiteSpace(rootPath))
340
                throw new ArgumentNullException("rootPath");
341
            if (!Path.IsPathRooted(rootPath))
342
                throw new ArgumentException("rootPath");
343
            if (String.IsNullOrWhiteSpace(folderName))
344
                throw new ArgumentNullException("folderName");
345
            Contract.EndContractBlock();
346

    
347
            var folder = Path.Combine(rootPath, folderName);
348
            if (!Directory.Exists(folder))
349
            {
350
                var info = Directory.CreateDirectory(folder);
351
                info.Attributes |= FileAttributes.Hidden;
352

    
353
                Log.InfoFormat("Created cache Folder: {0}", folder);
354
            }
355
            else
356
            {
357
                var info = new DirectoryInfo(folder);
358
                if ((info.Attributes & FileAttributes.Hidden) == 0)
359
                {
360
                    info.Attributes |= FileAttributes.Hidden;
361
                    Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
362
                }                                
363
            }
364
        }
365

    
366
       
367

    
368

    
369
        private void StartWatcherAgent()
370
        {
371
            if (Log.IsDebugEnabled)
372
                Log.DebugFormat("Start Folder Monitoring [{0}]",RootPath);
373

    
374
            AgentLocator<FileAgent>.Register(FileAgent,RootPath);
375
            
376
            FileAgent.IdleTimeout = Settings.FileIdleTimeout;
377
            FileAgent.StatusKeeper = StatusKeeper;
378
            FileAgent.StatusNotification = StatusNotification;
379
            FileAgent.Workflow = Workflow;
380
            FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
381
            FileAgent.Start(_accountInfo, RootPath);
382
        }
383

    
384
        public void Stop()
385
        {
386
            AgentLocator<FileAgent>.Remove(RootPath);
387

    
388
            if (FileAgent!=null)
389
                FileAgent.Stop();
390
            FileAgent = null;
391
        }
392

    
393

    
394
        ~PithosMonitor()
395
        {
396
            Dispose(false);
397
        }
398

    
399
        public void Dispose()
400
        {
401
            Dispose(true);
402
            GC.SuppressFinalize(this);
403
        }
404

    
405
        protected virtual void Dispose(bool disposing)
406
        {
407
            if (disposing)
408
            {
409
                Stop();
410
            }
411
        }
412

    
413

    
414
        public void MoveFileStates(string oldPath, string newPath)
415
        {
416
            if (String.IsNullOrWhiteSpace(oldPath))
417
                throw new ArgumentNullException("oldPath");
418
            if (!Path.IsPathRooted(oldPath))
419
                throw new ArgumentException("oldPath must be an absolute path","oldPath");
420
            if (string.IsNullOrWhiteSpace(newPath))
421
                throw new ArgumentNullException("newPath");
422
            if (!Path.IsPathRooted(newPath))
423
                throw new ArgumentException("newPath must be an absolute path","newPath");
424
            Contract.EndContractBlock();
425

    
426
            StatusKeeper.ChangeRoots(oldPath, newPath);
427
        }
428

    
429
        public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed)
430
        {
431
            //Convert the uris to paths
432
            var selectivePaths = UrisToFilePaths(uris);
433
            
434
            FileAgent.SelectivePaths=selectivePaths;
435
            PollAgent.SetSyncUris(uris);
436
            
437
            var removedPaths = UrisToFilePaths(removed);
438
            UnversionSelectivePaths(removedPaths);
439

    
440
        }
441

    
442
        /// <summary>
443
        /// Mark all unselected paths as Unversioned
444
        /// </summary>
445
        /// <param name="removed"></param>
446
        private void UnversionSelectivePaths(List<string> removed)
447
        {
448
            if (removed == null)
449
                return;
450

    
451
            //Ensure we remove any file state below the deleted folders
452
            FileState.UnversionPaths(removed);
453
        }
454

    
455

    
456
        /// <summary>
457
        /// Return a list of absolute filepaths from a list of Uris
458
        /// </summary>
459
        /// <param name="uris"></param>
460
        /// <returns></returns>
461
        private List<string> UrisToFilePaths(IEnumerable<Uri> uris)
462
        {
463
            if (uris == null)
464
                return new List<string>();
465

    
466
            return (from uri in uris
467
                    let relativePath = _accountInfo.StorageUri
468
                        .MakeRelativeUri(uri)
469
                        .RelativeUriToFilePath()
470
                        //Trim the account name
471
                    select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList();            
472
        }
473

    
474

    
475
        public ObjectInfo GetObjectInfo(string filePath)
476
        {
477
            if (String.IsNullOrWhiteSpace(filePath))
478
                throw new ArgumentNullException("filePath");
479
            Contract.EndContractBlock();
480

    
481
            var file=new FileInfo(filePath);
482
            string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
483
            var relativePath = file.AsRelativeTo(RootPath);
484
            
485
            string accountName,container;
486
            
487
            var parts=relativePath.Split('\\');
488

    
489
            var accountInfo = _accountInfo;
490
            if (relativePath.StartsWith(FolderConstants.OthersFolder))
491
            {                
492
                accountName = parts[1];
493
                container = parts[2];
494
                relativeUrl = String.Join("/", parts.Splice(3));
495
                //Create the root URL for the target account
496
                var oldName = UserName;
497
                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
498
                var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
499
                var root=absoluteUri.Substring(0, nameIndex);
500

    
501
                accountInfo = new AccountInfo
502
                {
503
                    UserName = accountName,
504
                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
505
                    StorageUri = new Uri(root + accountName),
506
                    BlockHash=accountInfo.BlockHash,
507
                    BlockSize=accountInfo.BlockSize,
508
                    Token=accountInfo.Token
509
                };
510
            }
511
            else
512
            {
513
                accountName = UserName;
514
                container = parts[0];
515
                relativeUrl = String.Join("/", parts.Splice(1));
516
            }
517
            
518
            var client = new CloudFilesClient(accountInfo);
519
            var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
520
            return objectInfo;
521
        }
522
        
523
        public Task<ContainerInfo> GetContainerInfo(string filePath)
524
        {
525
            if (String.IsNullOrWhiteSpace(filePath))
526
                throw new ArgumentNullException("filePath");
527
            Contract.EndContractBlock();
528

    
529
            var file=new FileInfo(filePath);
530
            var relativePath = file.AsRelativeTo(RootPath);
531
            
532
            string accountName,container;
533
            
534
            var parts=relativePath.Split('\\');
535

    
536
            var accountInfo = _accountInfo;
537
            if (relativePath.StartsWith(FolderConstants.OthersFolder))
538
            {                
539
                accountName = parts[1];
540
                container = parts[2];                
541
                //Create the root URL for the target account
542
                var oldName = UserName;
543
                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
544
                var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
545
                var root=absoluteUri.Substring(0, nameIndex);
546

    
547
                accountInfo = new AccountInfo
548
                {
549
                    UserName = accountName,
550
                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
551
                    StorageUri = new Uri(root + accountName),
552
                    BlockHash=accountInfo.BlockHash,
553
                    BlockSize=accountInfo.BlockSize,
554
                    Token=accountInfo.Token
555
                };
556
            }
557
            else
558
            {
559
                accountName = UserName;
560
                container = parts[0];                
561
            }
562

    
563
            return Task.Factory.StartNew(() =>
564
            {
565
                var client = new CloudFilesClient(accountInfo);
566
                var containerInfo = client.GetContainerInfo(accountName, container);
567
                return containerInfo;
568
            });
569
        }
570
    }
571
}