Statistics
| Branch: | Revision:

root / trunk / Pithos.Core / PithosMonitor.cs @ 6bcdd8e2

History | View | Annotate | Download (20.6 kB)

1
#region
2
/* -----------------------------------------------------------------------
3
 * <copyright file="PithosMonitor.cs" company="GRNet">
4
 * 
5
 * Copyright 2011-2012 GRNET S.A. All rights reserved.
6
 *
7
 * Redistribution and use in source and binary forms, with or
8
 * without modification, are permitted provided that the following
9
 * conditions are met:
10
 *
11
 *   1. Redistributions of source code must retain the above
12
 *      copyright notice, this list of conditions and the following
13
 *      disclaimer.
14
 *
15
 *   2. Redistributions in binary form must reproduce the above
16
 *      copyright notice, this list of conditions and the following
17
 *      disclaimer in the documentation and/or other materials
18
 *      provided with the distribution.
19
 *
20
 *
21
 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32
 * POSSIBILITY OF SUCH DAMAGE.
33
 *
34
 * The views and conclusions contained in the software and
35
 * documentation are those of the authors and should not be
36
 * interpreted as representing official policies, either expressed
37
 * or implied, of GRNET S.A.
38
 * </copyright>
39
 * -----------------------------------------------------------------------
40
 */
41
#endregion
42
using System;
43
using System.Collections.Generic;
44
using System.ComponentModel.Composition;
45
using System.Diagnostics.Contracts;
46
using System.IO;
47
using System.Linq;
48
using System.Reflection;
49
using System.Threading;
50
using System.Threading.Tasks;
51
using Pithos.Core.Agents;
52
using Pithos.Interfaces;
53
using Pithos.Network;
54
using log4net;
55

    
56
namespace Pithos.Core
57
{
58
    [Export(typeof(PithosMonitor))]
59
    public class PithosMonitor:IDisposable
60
    {
61
        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
62

    
63
        private int _blockSize;
64
        private string _blockHash;
65

    
66
        [Import]
67
        public IPithosSettings Settings{get;set;}
68

    
69
        private IStatusKeeper _statusKeeper;
70

    
71
        [Import]
72
        public IStatusKeeper StatusKeeper
73
        {
74
            get { return _statusKeeper; }
75
            set
76
            {
77
                _statusKeeper = value;
78
                FileAgent.StatusKeeper = value;
79
            }
80
        }
81

    
82

    
83
        private IPithosWorkflow _workflow;
84

    
85
        [Import]
86
        public IPithosWorkflow Workflow
87
        {
88
            get { return _workflow; }
89
            set
90
            {
91
                _workflow = value;
92
                FileAgent.Workflow = value;
93
            }
94
        }
95

    
96
        public ICloudClient CloudClient { get; set; }
97

    
98
        public IStatusNotification StatusNotification { get; set; }
99

    
100
        //[Import]
101
        public FileAgent FileAgent { get; private set; }
102

    
103
        private WorkflowAgent _workflowAgent;
104

    
105
        [Import]
106
        public WorkflowAgent WorkflowAgent
107
        {
108
            get { return _workflowAgent; }
109
            set
110
            {
111
                _workflowAgent = value;
112
                FileAgent.WorkflowAgent = value;
113
            }
114
        }
115
        
116
        [Import]
117
        public NetworkAgent NetworkAgent { get; set; }
118
        [Import]
119
        public PollAgent PollAgent { get; set; }       
120

    
121
        public string UserName { get; set; }
122
        private string _apiKey;
123
        public string ApiKey
124
        {
125
            get { return _apiKey; }
126
            set
127
            {
128
                _apiKey = value;
129
                if (_accountInfo != null)
130
                    _accountInfo.Token = value;
131
            }
132
        }
133

    
134
        private AccountInfo _accountInfo;
135

    
136

    
137

    
138

    
139

    
140

    
141
        public bool Pause
142
        {
143
            get { return FileAgent.Pause; }
144
            set
145
            {
146
                FileAgent.Pause = value;
147
                if (value)
148
                {
149
                    StatusKeeper.SetPithosStatus(PithosStatus.SyncPaused);
150
                    StatusNotification.NotifyChange("Paused");
151
                }
152
                else
153
                {
154
                    StatusKeeper.SetPithosStatus(PithosStatus.InSynch);
155
                    StatusNotification.NotifyChange("Synchronizing");
156
                }
157
            }
158
        }
159

    
160
        private string _rootPath;
161
        public string RootPath
162
        {
163
            get { return _rootPath; }
164
            set 
165
            {
166
                _rootPath = String.IsNullOrWhiteSpace(value) 
167
                    ? String.Empty 
168
                    : value.ToLower();
169
            }
170
        }
171

    
172

    
173
        CancellationTokenSource _cancellationSource;
174

    
175
        public PithosMonitor()
176
        {
177
            FileAgent = new FileAgent();
178

    
179
        }
180
        private bool _started;
181

    
182
        public void Start()
183
        {            
184
            if (String.IsNullOrWhiteSpace(ApiKey))
185
                throw new InvalidOperationException("The ApiKey is empty");
186
            if (String.IsNullOrWhiteSpace(UserName))
187
                throw new InvalidOperationException("The UserName is empty");
188
            if (String.IsNullOrWhiteSpace(AuthenticationUrl))
189
                throw new InvalidOperationException("The Authentication url is empty");
190
            Contract.EndContractBlock();
191

    
192
            //If the account doesn't have a valid path, don't start monitoring but don't throw either
193
            if (String.IsNullOrWhiteSpace(RootPath))
194
                //TODO; Warn user?
195
                return;
196

    
197
            WorkflowAgent.StatusNotification = StatusNotification;
198

    
199
            StatusNotification.NotifyChange("Starting");
200
            if (_started)
201
            {
202
                if (!_cancellationSource.IsCancellationRequested)
203
                    return;
204
            }
205
            _cancellationSource = new CancellationTokenSource();
206

    
207

    
208
            CloudClient = new CloudFilesClient(UserName, ApiKey)
209
                              {UsePithos = true, AuthenticationUrl = AuthenticationUrl};
210

    
211

    
212
            _accountInfo = CloudClient.Authenticate();            
213
            _accountInfo.SiteUri = AuthenticationUrl;
214
            _accountInfo.AccountPath = RootPath;
215

    
216

    
217
            var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
218
            if (!Directory.Exists(pithosFolder))
219
                Directory.CreateDirectory(pithosFolder);
220
            //Create the cache folder and ensure it is hidden
221
            CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
222

    
223
            var policy=CloudClient.GetAccountPolicies(_accountInfo);
224

    
225
            StatusNotification.NotifyAccount(policy);
226
            EnsurePithosContainers();
227
            
228
            StatusKeeper.BlockHash = _blockHash;
229
            StatusKeeper.BlockSize = _blockSize;
230
            
231
            StatusKeeper.StartProcessing(_cancellationSource.Token);
232
            IndexLocalFiles();
233
            //Extract the URIs from the string collection
234
            var settings = Settings.Accounts.First(s => s.AccountName == _accountInfo.UserName);
235
            var selectiveUrls=settings.SelectiveFolders.Cast<string>().Select(url => new Uri(url)).ToArray();
236

    
237
            SetSelectivePaths(selectiveUrls,null,null);
238
            
239
            StartWatcherAgent();
240

    
241
            StartNetworkAgent();
242
            
243
            WorkflowAgent.RestartInterruptedFiles(_accountInfo);
244
            _started = true;
245
        }
246

    
247
        private void EnsurePithosContainers()
248
        {
249

    
250
            //Create the two default containers if they are missing
251
            var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
252
            foreach (var container in pithosContainers)
253
            {                
254
                var info=CloudClient.GetContainerInfo(UserName, container);
255
                if (info == ContainerInfo.Empty)
256
                {
257
                    CloudClient.CreateContainer(UserName, container);
258
                    info = CloudClient.GetContainerInfo(UserName, container);
259
                }
260
                _blockSize = info.BlockSize;
261
                _blockHash = info.BlockHash;
262
                _accountInfo.BlockSize = _blockSize;
263
                _accountInfo.BlockHash = _blockHash;
264
            }
265
        }
266

    
267
        public string AuthenticationUrl { get; set; }
268

    
269
        private void IndexLocalFiles()
270
        {
271
            StatusNotification.NotifyChange("Indexing Local Files");
272
            using (ThreadContext.Stacks["Operation"].Push("Indexing local files"))
273
            {
274
                Log.Info("START");
275
                try
276
                {
277
                    var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
278
                    var directory = new DirectoryInfo(RootPath);
279
                    var files =
280
                        from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
281
                        where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
282
                              !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
283
                        select file;
284
                    StatusKeeper.ProcessExistingFiles(files);
285

    
286
                }
287
                catch (Exception exc)
288
                {
289
                    Log.Error("[ERROR]", exc);
290
                }
291
                finally
292
                {
293
                    Log.Info("[END]");
294
                }
295
            }
296
        }
297

    
298
        
299
  
300

    
301

    
302
       /* private void StartWorkflowAgent()
303
        {
304
            WorkflowAgent.StatusNotification = StatusNotification;
305

    
306
/*            //On Vista and up we can check for a network connection
307
            bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
308
            //If we are not connected retry later
309
            if (!connected)
310
            {
311
                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
312
                return;
313
            }#1#
314

    
315
            try
316
            {
317
                WorkflowAgent.Start();                
318
            }
319
            catch (Exception)
320
            {
321
                //Faild to authenticate due to network or account error
322
                //Retry after a while
323
                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
324
            }
325
        }*/
326

    
327
        internal class LocalFileComparer:EqualityComparer<CloudAction>
328
        {
329
            public override bool Equals(CloudAction x, CloudAction y)
330
            {
331
                if (x.Action != y.Action)
332
                    return false;
333
                if (x.LocalFile != null && y.LocalFile != null && !x.LocalFile.FullName.Equals(y.LocalFile.FullName))
334
                    return false;
335
                if (x.CloudFile != null && y.CloudFile != null )
336
                {
337
                    if (x.CloudFile.Hash == null & y.CloudFile.Hash != null)
338
                        return false;
339
                    if (x.CloudFile.Hash != null & y.CloudFile.Hash == null)
340
                        return false;
341
                    if (x.CloudFile.Hash == null & y.CloudFile.Hash == null)
342
                        return (x.CloudFile.Name == y.CloudFile.Name);
343
                    if (!x.CloudFile.Hash.Equals(y.CloudFile.Hash))
344
                        return false;
345
                }
346
                if (x.CloudFile == null ^ y.CloudFile == null ||
347
                    x.LocalFile == null ^ y.LocalFile == null)
348
                    return false;
349
                return true;
350
            }
351

    
352
            public override int GetHashCode(CloudAction obj)
353
            {
354
                if (obj == null)
355
                    return 0;
356
                var hash1 = (obj.LocalFile == null) ? int.MaxValue : obj.LocalFile.FullName.GetHashCode();
357
                var hash2 = (obj.CloudFile == null) ? int.MaxValue : (obj.CloudFile.Hash ?? obj.CloudFile.Name??"").GetHashCode();
358
                var hash3 = obj.Action.GetHashCode();
359
                return hash1 ^ hash2 & hash3;
360
            }
361
        }        
362

    
363
        private void StartNetworkAgent()
364
        {
365

    
366
            NetworkAgent.StatusNotification = StatusNotification;
367
                        
368
            NetworkAgent.Start();
369

    
370
            PollAgent.AddAccount(_accountInfo);
371

    
372
            PollAgent.StatusNotification = StatusNotification;
373

    
374
            PollAgent.PollRemoteFiles();
375
        }
376

    
377
        //Make sure a hidden cache folder exists to store partial downloads
378
        private static void CreateHiddenFolder(string rootPath, string folderName)
379
        {
380
            if (String.IsNullOrWhiteSpace(rootPath))
381
                throw new ArgumentNullException("rootPath");
382
            if (!Path.IsPathRooted(rootPath))
383
                throw new ArgumentException("rootPath");
384
            if (String.IsNullOrWhiteSpace(folderName))
385
                throw new ArgumentNullException("folderName");
386
            Contract.EndContractBlock();
387

    
388
            var folder = Path.Combine(rootPath, folderName);
389
            if (!Directory.Exists(folder))
390
            {
391
                var info = Directory.CreateDirectory(folder);
392
                info.Attributes |= FileAttributes.Hidden;
393

    
394
                Log.InfoFormat("Created cache Folder: {0}", folder);
395
            }
396
            else
397
            {
398
                var info = new DirectoryInfo(folder);
399
                if ((info.Attributes & FileAttributes.Hidden) == 0)
400
                {
401
                    info.Attributes |= FileAttributes.Hidden;
402
                    Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
403
                }                                
404
            }
405
        }
406

    
407
       
408

    
409

    
410
        private void StartWatcherAgent()
411
        {
412
            AgentLocator<FileAgent>.Register(FileAgent,RootPath);
413

    
414
            FileAgent.StatusKeeper = StatusKeeper;
415
            FileAgent.Workflow = Workflow;
416
            FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
417
            FileAgent.Start(_accountInfo, RootPath);
418
        }
419

    
420
        public void Stop()
421
        {
422
            AgentLocator<FileAgent>.Remove(RootPath);
423

    
424
            if (FileAgent!=null)
425
                FileAgent.Stop();
426
            FileAgent = null;
427
        }
428

    
429

    
430
        ~PithosMonitor()
431
        {
432
            Dispose(false);
433
        }
434

    
435
        public void Dispose()
436
        {
437
            Dispose(true);
438
            GC.SuppressFinalize(this);
439
        }
440

    
441
        protected virtual void Dispose(bool disposing)
442
        {
443
            if (disposing)
444
            {
445
                Stop();
446
            }
447
        }
448

    
449

    
450
        public void MoveFileStates(string oldPath, string newPath)
451
        {
452
            if (String.IsNullOrWhiteSpace(oldPath))
453
                throw new ArgumentNullException("oldPath");
454
            if (!Path.IsPathRooted(oldPath))
455
                throw new ArgumentException("oldPath must be an absolute path","oldPath");
456
            if (string.IsNullOrWhiteSpace(newPath))
457
                throw new ArgumentNullException("newPath");
458
            if (!Path.IsPathRooted(newPath))
459
                throw new ArgumentException("newPath must be an absolute path","newPath");
460
            Contract.EndContractBlock();
461

    
462
            StatusKeeper.ChangeRoots(oldPath, newPath);
463
        }
464

    
465
        public void SetSelectivePaths(Uri[] uris,Uri[] added, Uri[] removed)
466
        {
467
            //Convert the uris to paths
468
            var selectivePaths = UrisToFilePaths(uris);
469
            
470
            FileAgent.SelectivePaths=selectivePaths;
471
            PollAgent.SetSyncUris(uris);
472
            
473
            var removedPaths = UrisToFilePaths(removed);
474
            UnversionSelectivePaths(removedPaths);
475

    
476
        }
477

    
478
        /// <summary>
479
        /// Mark all unselected paths as Unversioned
480
        /// </summary>
481
        /// <param name="removed"></param>
482
        private void UnversionSelectivePaths(List<string> removed)
483
        {
484
            if (removed == null)
485
                return;
486

    
487
            //Ensure we remove any file state below the deleted folders
488
            FileState.UnversionPaths(removed);
489
        }
490

    
491

    
492
        /// <summary>
493
        /// Return a list of absolute filepaths from a list of Uris
494
        /// </summary>
495
        /// <param name="uris"></param>
496
        /// <returns></returns>
497
        private List<string> UrisToFilePaths(IEnumerable<Uri> uris)
498
        {
499
            if (uris == null)
500
                return new List<string>();
501

    
502
            return (from uri in uris
503
                    let relativePath = _accountInfo.StorageUri
504
                        .MakeRelativeUri(uri)
505
                        .RelativeUriToFilePath()
506
                        //Trim the account name
507
                    select Path.Combine(RootPath, relativePath.After(_accountInfo.UserName + '\\'))).ToList();            
508
        }
509

    
510

    
511
        public ObjectInfo GetObjectInfo(string filePath)
512
        {
513
            if (String.IsNullOrWhiteSpace(filePath))
514
                throw new ArgumentNullException("filePath");
515
            Contract.EndContractBlock();
516

    
517
            var file=new FileInfo(filePath);
518
            string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
519
            var relativePath = file.AsRelativeTo(RootPath);
520
            
521
            string accountName,container;
522
            
523
            var parts=relativePath.Split('\\');
524

    
525
            var accountInfo = _accountInfo;
526
            if (relativePath.StartsWith(FolderConstants.OthersFolder))
527
            {                
528
                accountName = parts[1];
529
                container = parts[2];
530
                relativeUrl = String.Join("/", parts.Splice(3));
531
                //Create the root URL for the target account
532
                var oldName = UserName;
533
                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
534
                var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
535
                var root=absoluteUri.Substring(0, nameIndex);
536

    
537
                accountInfo = new AccountInfo
538
                {
539
                    UserName = accountName,
540
                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
541
                    StorageUri = new Uri(root + accountName),
542
                    BlockHash=accountInfo.BlockHash,
543
                    BlockSize=accountInfo.BlockSize,
544
                    Token=accountInfo.Token
545
                };
546
            }
547
            else
548
            {
549
                accountName = UserName;
550
                container = parts[0];
551
                relativeUrl = String.Join("/", parts.Splice(1));
552
            }
553
            
554
            var client = new CloudFilesClient(accountInfo);
555
            var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
556
            return objectInfo;
557
        }
558
        
559
        public Task<ContainerInfo> GetContainerInfo(string filePath)
560
        {
561
            if (String.IsNullOrWhiteSpace(filePath))
562
                throw new ArgumentNullException("filePath");
563
            Contract.EndContractBlock();
564

    
565
            var file=new FileInfo(filePath);
566
            var relativePath = file.AsRelativeTo(RootPath);
567
            
568
            string accountName,container;
569
            
570
            var parts=relativePath.Split('\\');
571

    
572
            var accountInfo = _accountInfo;
573
            if (relativePath.StartsWith(FolderConstants.OthersFolder))
574
            {                
575
                accountName = parts[1];
576
                container = parts[2];                
577
                //Create the root URL for the target account
578
                var oldName = UserName;
579
                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
580
                var nameIndex=absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
581
                var root=absoluteUri.Substring(0, nameIndex);
582

    
583
                accountInfo = new AccountInfo
584
                {
585
                    UserName = accountName,
586
                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
587
                    StorageUri = new Uri(root + accountName),
588
                    BlockHash=accountInfo.BlockHash,
589
                    BlockSize=accountInfo.BlockSize,
590
                    Token=accountInfo.Token
591
                };
592
            }
593
            else
594
            {
595
                accountName = UserName;
596
                container = parts[0];                
597
            }
598

    
599
            return Task.Factory.StartNew(() =>
600
            {
601
                var client = new CloudFilesClient(accountInfo);
602
                var containerInfo = client.GetContainerInfo(accountName, container);
603
                return containerInfo;
604
            });
605
        }
606
    }
607
}