Statistics
| Branch: | Revision:

root / trunk / Pithos.Core / PithosMonitor.cs @ 759bd3c4

History | View | Annotate | Download (20.6 kB)

1
#region
2
/* -----------------------------------------------------------------------
3
 * <copyright file="PithosMonitor.cs" company="GRNet">
4
 * 
5
 * Copyright 2011-2012 GRNET S.A. All rights reserved.
6
 *
7
 * Redistribution and use in source and binary forms, with or
8
 * without modification, are permitted provided that the following
9
 * conditions are met:
10
 *
11
 *   1. Redistributions of source code must retain the above
12
 *      copyright notice, this list of conditions and the following
13
 *      disclaimer.
14
 *
15
 *   2. Redistributions in binary form must reproduce the above
16
 *      copyright notice, this list of conditions and the following
17
 *      disclaimer in the documentation and/or other materials
18
 *      provided with the distribution.
19
 *
20
 *
21
 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32
 * POSSIBILITY OF SUCH DAMAGE.
33
 *
34
 * The views and conclusions contained in the software and
35
 * documentation are those of the authors and should not be
36
 * interpreted as representing official policies, either expressed
37
 * or implied, of GRNET S.A.
38
 * </copyright>
39
 * -----------------------------------------------------------------------
40
 */
41
#endregion
42
using System;
43
using System.Collections.Concurrent;
44
using System.Collections.Generic;
45
using System.ComponentModel.Composition;
46
using System.Diagnostics;
47
using System.Diagnostics.Contracts;
48
using System.IO;
49
using System.Linq;
50
using System.Net;
51
using System.Net.NetworkInformation;
52
using System.Security.Cryptography;
53
using System.ServiceModel.Description;
54
using System.Text;
55
using System.Threading;
56
using System.Threading.Tasks;
57
using Castle.ActiveRecord.Queries;
58
using Microsoft.WindowsAPICodePack.Net;
59
using Pithos.Core.Agents;
60
using Pithos.Interfaces;
61
using System.ServiceModel;
62
using Pithos.Network;
63
using log4net;
64

    
65
namespace Pithos.Core
66
{
67
    [Export(typeof(PithosMonitor))]
68
    public class PithosMonitor:IDisposable
69
    {
70
        private int _blockSize;
71
        private string _blockHash;
72

    
73
        [Import]
74
        public IPithosSettings Settings{get;set;}
75

    
76
        private IStatusKeeper _statusKeeper;
77

    
78
        [Import]
79
        public IStatusKeeper StatusKeeper
80
        {
81
            get { return _statusKeeper; }
82
            set
83
            {
84
                _statusKeeper = value;
85
                FileAgent.StatusKeeper = value;
86
            }
87
        }
88

    
89

    
90
        private IPithosWorkflow _workflow;
91

    
92
        [Import]
93
        public IPithosWorkflow Workflow
94
        {
95
            get { return _workflow; }
96
            set
97
            {
98
                _workflow = value;
99
                FileAgent.Workflow = value;
100
            }
101
        }
102

    
103
        public ICloudClient CloudClient { get; set; }
104

    
105
        public IStatusNotification StatusNotification { get; set; }
106

    
107
        //[Import]
108
        public FileAgent FileAgent { get; private set; }
109

    
110
        private WorkflowAgent _workflowAgent;
111

    
112
        [Import]
113
        public WorkflowAgent WorkflowAgent
114
        {
115
            get { return _workflowAgent; }
116
            set
117
            {
118
                _workflowAgent = value;
119
                FileAgent.WorkflowAgent = value;
120
            }
121
        }
122
        
123
        [Import]
124
        public NetworkAgent NetworkAgent { get; set; }
125
        [Import]
126
        public PollAgent PollAgent { get; set; }       
127

    
128
        public string UserName { get; set; }
129
        private string _apiKey;
130
        public string ApiKey
131
        {
132
            get { return _apiKey; }
133
            set
134
            {
135
                _apiKey = value;
136
                if (_accountInfo != null)
137
                    _accountInfo.Token = value;
138
            }
139
        }
140

    
141
        private AccountInfo _accountInfo;
142
        
143
       
144

    
145

    
146
        private static readonly ILog Log = LogManager.GetLogger(typeof(PithosMonitor));
147

    
148

    
149
        public bool Pause
150
        {
151
            get { return FileAgent.Pause; }
152
            set
153
            {
154
                FileAgent.Pause = value;
155
                if (value)
156
                {
157
                    StatusKeeper.SetPithosStatus(PithosStatus.SyncPaused);
158
                    StatusNotification.NotifyChange("Paused");
159
                }
160
                else
161
                {
162
                    StatusKeeper.SetPithosStatus(PithosStatus.InSynch);
163
                    StatusNotification.NotifyChange("Synchronizing");
164
                }
165
            }
166
        }
167

    
168
        private string _rootPath;
169
        public string RootPath
170
        {
171
            get { return _rootPath; }
172
            set 
173
            {
174
                _rootPath = String.IsNullOrWhiteSpace(value) 
175
                    ? String.Empty 
176
                    : value.ToLower();
177
            }
178
        }
179

    
180

    
181
        CancellationTokenSource _cancellationSource;
182

    
183
        public PithosMonitor()
184
        {
185
            FileAgent = new FileAgent();
186

    
187
        }
188
        private bool _started;
189

    
190
        public void Start()
191
        {            
192
            if (String.IsNullOrWhiteSpace(ApiKey))
193
                throw new InvalidOperationException("The ApiKey is empty");
194
            if (String.IsNullOrWhiteSpace(UserName))
195
                throw new InvalidOperationException("The UserName is empty");
196
            if (String.IsNullOrWhiteSpace(AuthenticationUrl))
197
                throw new InvalidOperationException("The Authentication url is empty");
198
            Contract.EndContractBlock();
199

    
200
            //If the account doesn't have a valid path, don't start monitoring but don't throw either
201
            if (String.IsNullOrWhiteSpace(RootPath))
202
                //TODO; Warn user?
203
                return;
204

    
205
            WorkflowAgent.StatusNotification = StatusNotification;
206

    
207
            StatusNotification.NotifyChange("Starting");
208
            if (_started)
209
            {
210
                if (!_cancellationSource.IsCancellationRequested)
211
                    return;
212
            }
213
            _cancellationSource = new CancellationTokenSource();
214
            
215

    
216
            CloudClient=new CloudFilesClient(UserName,ApiKey);
217
            CloudClient.UsePithos = true;
218
            CloudClient.AuthenticationUrl = this.AuthenticationUrl;            
219

    
220
            _accountInfo = CloudClient.Authenticate();            
221
            _accountInfo.SiteUri = AuthenticationUrl;
222
            _accountInfo.AccountPath = RootPath;
223

    
224

    
225
            var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
226
            if (!Directory.Exists(pithosFolder))
227
                Directory.CreateDirectory(pithosFolder);
228
            //Create the cache folder and ensure it is hidden
229
            CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
230

    
231
            var policy=CloudClient.GetAccountPolicies(_accountInfo);
232

    
233
            StatusNotification.NotifyAccount(policy);
234
            EnsurePithosContainers();
235
            
236
            StatusKeeper.BlockHash = _blockHash;
237
            StatusKeeper.BlockSize = _blockSize;
238
            
239
            StatusKeeper.StartProcessing(_cancellationSource.Token);
240
            IndexLocalFiles();
241
            //Extract the URIs from the string collection
242
            var settings = Settings.Accounts.First(s => s.AccountName == _accountInfo.UserName);
243
            var selectiveUrls=new string[settings.SelectiveFolders.Count];
244
            settings.SelectiveFolders.CopyTo(selectiveUrls,0);
245

    
246
            SetSelectivePaths(selectiveUrls,null,null);
247
            
248
            StartWatcherAgent();
249

    
250
            StartNetworkAgent();
251
            
252
            WorkflowAgent.RestartInterruptedFiles(_accountInfo);
253
            _started = true;
254
        }
255

    
256
        private void EnsurePithosContainers()
257
        {
258

    
259
            //Create the two default containers if they are missing
260
            var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
261
            foreach (var container in pithosContainers)
262
            {                
263
                var info=CloudClient.GetContainerInfo(this.UserName, container);
264
                if (info == ContainerInfo.Empty)
265
                {
266
                    CloudClient.CreateContainer(this.UserName, container);
267
                    info = CloudClient.GetContainerInfo(this.UserName, container);
268
                }
269
                _blockSize = info.BlockSize;
270
                _blockHash = info.BlockHash;
271
                _accountInfo.BlockSize = _blockSize;
272
                _accountInfo.BlockHash = _blockHash;
273
            }
274
        }
275

    
276
        public string AuthenticationUrl { get; set; }
277

    
278
        private void IndexLocalFiles()
279
        {
280
            StatusNotification.NotifyChange("Indexing Local Files",TraceLevel.Info);
281
            using (log4net.ThreadContext.Stacks["Monitor"].Push("Indexing local files"))
282
            {
283
                Log.Info("START");
284
                try
285
                {
286
                    var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
287
                    var directory = new DirectoryInfo(RootPath);
288
                    var files =
289
                        from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
290
                        where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
291
                              !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
292
                        select file;
293
                    StatusKeeper.ProcessExistingFiles(files);
294

    
295
                }
296
                catch (Exception exc)
297
                {
298
                    Log.Error("[ERROR]", exc);
299
                }
300
                finally
301
                {
302
                    Log.Info("[END]");
303
                }
304
            }
305
        }
306

    
307
        
308
  
309

    
310

    
311
       /* private void StartWorkflowAgent()
312
        {
313
            WorkflowAgent.StatusNotification = StatusNotification;
314

    
315
/*            //On Vista and up we can check for a network connection
316
            bool connected=Environment.OSVersion.Version.Major < 6 || NetworkListManager.IsConnectedToInternet;
317
            //If we are not connected retry later
318
            if (!connected)
319
            {
320
                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
321
                return;
322
            }#1#
323

    
324
            try
325
            {
326
                WorkflowAgent.Start();                
327
            }
328
            catch (Exception)
329
            {
330
                //Faild to authenticate due to network or account error
331
                //Retry after a while
332
                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
333
            }
334
        }*/
335

    
336
        internal class LocalFileComparer:EqualityComparer<CloudAction>
337
        {
338
            public override bool Equals(CloudAction x, CloudAction y)
339
            {
340
                if (x.Action != y.Action)
341
                    return false;
342
                if (x.LocalFile != null && y.LocalFile != null && !x.LocalFile.FullName.Equals(y.LocalFile.FullName))
343
                    return false;
344
                if (x.CloudFile != null && y.CloudFile != null )
345
                {
346
                    if (x.CloudFile.Hash == null & y.CloudFile.Hash != null)
347
                        return false;
348
                    if (x.CloudFile.Hash != null & y.CloudFile.Hash == null)
349
                        return false;
350
                    if (x.CloudFile.Hash == null & y.CloudFile.Hash == null)
351
                        return (x.CloudFile.Name == y.CloudFile.Name);
352
                    if (!x.CloudFile.Hash.Equals(y.CloudFile.Hash))
353
                        return false;
354
                }
355
                if (x.CloudFile == null ^ y.CloudFile == null ||
356
                    x.LocalFile == null ^ y.LocalFile == null)
357
                    return false;
358
                return true;
359
            }
360

    
361
            public override int GetHashCode(CloudAction obj)
362
            {
363
                var hash1 = (obj.LocalFile == null) ? int.MaxValue : obj.LocalFile.FullName.GetHashCode();
364
                var hash2 = (obj.CloudFile == null) ? int.MaxValue : (obj.CloudFile.Hash ?? obj.CloudFile.Name??"").GetHashCode();
365
                var hash3 = obj.Action.GetHashCode();
366
                return hash1 ^ hash2 & hash3;
367
            }
368
        }        
369

    
370
        private Timer timer;
371

    
372
        private void StartNetworkAgent()
373
        {
374

    
375
            NetworkAgent.AddAccount(_accountInfo);
376

    
377
            NetworkAgent.StatusNotification = StatusNotification;
378
                        
379
            NetworkAgent.Start();
380

    
381
            PollAgent.StatusNotification = StatusNotification;
382

    
383
            PollAgent.PollRemoteFiles();
384
        }
385

    
386
        //Make sure a hidden cache folder exists to store partial downloads
387
        private static string CreateHiddenFolder(string rootPath, string folderName)
388
        {
389
            if (String.IsNullOrWhiteSpace(rootPath))
390
                throw new ArgumentNullException("rootPath");
391
            if (!Path.IsPathRooted(rootPath))
392
                throw new ArgumentException("rootPath");
393
            if (String.IsNullOrWhiteSpace(folderName))
394
                throw new ArgumentNullException("folderName");
395
            Contract.EndContractBlock();
396

    
397
            var folder = Path.Combine(rootPath, folderName);
398
            if (!Directory.Exists(folder))
399
            {
400
                var info = Directory.CreateDirectory(folder);
401
                info.Attributes |= FileAttributes.Hidden;
402

    
403
                Log.InfoFormat("Created cache Folder: {0}", folder);
404
            }
405
            else
406
            {
407
                var info = new DirectoryInfo(folder);
408
                if ((info.Attributes & FileAttributes.Hidden) == 0)
409
                {
410
                    info.Attributes |= FileAttributes.Hidden;
411
                    Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
412
                }                                
413
            }
414
            return folder;
415
        }
416

    
417
       
418

    
419

    
420
        private void StartWatcherAgent()
421
        {
422
            AgentLocator<FileAgent>.Register(FileAgent,RootPath);
423

    
424
            FileAgent.StatusKeeper = StatusKeeper;
425
            FileAgent.Workflow = Workflow;
426
            FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
427
            FileAgent.Start(_accountInfo, RootPath);
428
        }
429

    
430
        public void Stop()
431
        {
432
            AgentLocator<FileAgent>.Remove(RootPath);
433

    
434
            if (FileAgent!=null)
435
                FileAgent.Stop();
436
            FileAgent = null;
437
            if (timer != null)
438
                timer.Dispose();
439
            timer = null;            
440
        }
441

    
442

    
443
        ~PithosMonitor()
444
        {
445
            Dispose(false);
446
        }
447

    
448
        public void Dispose()
449
        {
450
            Dispose(true);
451
            GC.SuppressFinalize(this);
452
        }
453

    
454
        protected virtual void Dispose(bool disposing)
455
        {
456
            if (disposing)
457
            {
458
                Stop();
459
            }
460
        }
461

    
462

    
463
        public void MoveFileStates(string oldPath, string newPath)
464
        {
465
            if (String.IsNullOrWhiteSpace(oldPath))
466
                throw new ArgumentNullException("oldPath");
467
            if (!Path.IsPathRooted(oldPath))
468
                throw new ArgumentException("oldPath must be an absolute path","oldPath");
469
            if (string.IsNullOrWhiteSpace(newPath))
470
                throw new ArgumentNullException("newPath");
471
            if (!Path.IsPathRooted(newPath))
472
                throw new ArgumentException("newPath must be an absolute path","newPath");
473
            Contract.EndContractBlock();
474

    
475
            StatusKeeper.ChangeRoots(oldPath, newPath);
476
        }
477

    
478
        public void SetSelectivePaths(string[] uris,string[] added, string[] removed)
479
        {
480
            //Convert the uris to paths
481
            var selectivePaths = (from string selectiveUrl in uris
482
                                    select new Uri(selectiveUrl)
483
                                    .MakeRelativeUri(_accountInfo.StorageUri)
484
                                    .RelativeUriToFilePath());
485

    
486
            FileAgent.SelectivePaths=selectivePaths.ToList();
487
            PollAgent.SetSyncUris(uris);
488
            RemoveSelectivePaths(removed);
489

    
490
        }
491

    
492
        /// <summary>
493
        /// Delete the folders that were removed from synchronization
494
        /// </summary>
495
        /// <param name="removed"></param>
496
        private void RemoveSelectivePaths(IEnumerable<string> removed)
497
        {
498
            if (removed == null)
499
                return;
500

    
501
            foreach (var removedPath in removed.Where(Directory.Exists))
502
            {
503
                Directory.Delete(removedPath,true);
504
            }
505

    
506
            //Ensure we remove any file state below the deleted folders
507
            FileState.RemovePaths(removed);
508
        }
509

    
510
        public ObjectInfo GetObjectInfo(string filePath)
511
        {
512
            if (String.IsNullOrWhiteSpace(filePath))
513
                throw new ArgumentNullException("filePath");
514
            Contract.EndContractBlock();
515

    
516
            var file=new FileInfo(filePath);
517
            string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
518
            var relativePath = file.AsRelativeTo(RootPath);
519
            
520
            string accountName,container;
521
            
522
            var parts=relativePath.Split('\\');
523

    
524
            var accountInfo = _accountInfo;
525
            if (relativePath.StartsWith(FolderConstants.OthersFolder))
526
            {                
527
                accountName = parts[1];
528
                container = parts[2];
529
                relativeUrl = String.Join("/", parts.Splice(3));
530
                //Create the root URL for the target account
531
                var oldName = UserName;
532
                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
533
                var nameIndex=absoluteUri.IndexOf(oldName);
534
                var root=absoluteUri.Substring(0, nameIndex);
535

    
536
                accountInfo = new AccountInfo
537
                {
538
                    UserName = accountName,
539
                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
540
                    StorageUri = new Uri(root + accountName),
541
                    BlockHash=accountInfo.BlockHash,
542
                    BlockSize=accountInfo.BlockSize,
543
                    Token=accountInfo.Token
544
                };
545
            }
546
            else
547
            {
548
                accountName = this.UserName;
549
                container = parts[0];
550
                relativeUrl = String.Join("/", parts.Splice(1));
551
            }
552
            
553
            var client = new CloudFilesClient(accountInfo);
554
            var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
555
            return objectInfo;
556
        }
557
        
558
        public Task<ContainerInfo> GetContainerInfo(string filePath)
559
        {
560
            if (String.IsNullOrWhiteSpace(filePath))
561
                throw new ArgumentNullException("filePath");
562
            Contract.EndContractBlock();
563

    
564
            var file=new FileInfo(filePath);
565
            var relativePath = file.AsRelativeTo(RootPath);
566
            
567
            string accountName,container;
568
            
569
            var parts=relativePath.Split('\\');
570

    
571
            var accountInfo = _accountInfo;
572
            if (relativePath.StartsWith(FolderConstants.OthersFolder))
573
            {                
574
                accountName = parts[1];
575
                container = parts[2];                
576
                //Create the root URL for the target account
577
                var oldName = UserName;
578
                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
579
                var nameIndex=absoluteUri.IndexOf(oldName);
580
                var root=absoluteUri.Substring(0, nameIndex);
581

    
582
                accountInfo = new AccountInfo
583
                {
584
                    UserName = accountName,
585
                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
586
                    StorageUri = new Uri(root + accountName),
587
                    BlockHash=accountInfo.BlockHash,
588
                    BlockSize=accountInfo.BlockSize,
589
                    Token=accountInfo.Token
590
                };
591
            }
592
            else
593
            {
594
                accountName = UserName;
595
                container = parts[0];                
596
            }
597

    
598
            return Task.Factory.StartNew(() =>
599
            {
600
                var client = new CloudFilesClient(accountInfo);
601
                var containerInfo = client.GetContainerInfo(accountName, container);
602
                return containerInfo;
603
            });
604
        }
605
    }
606
}