Statistics
| Branch: | Revision:

root / trunk / Pithos.Core / PithosMonitor.cs @ 23821bd2

History | View | Annotate | Download (18 kB)

1
using System;
2
using System.Collections.Concurrent;
3
using System.Collections.Generic;
4
using System.ComponentModel.Composition;
5
using System.Diagnostics;
6
using System.Diagnostics.Contracts;
7
using System.IO;
8
using System.Linq;
9
using System.Net.NetworkInformation;
10
using System.Security.Cryptography;
11
using System.ServiceModel.Description;
12
using System.Text;
13
using System.Threading;
14
using System.Threading.Tasks;
15
using Castle.ActiveRecord.Queries;
16
using Microsoft.WindowsAPICodePack.Net;
17
using Pithos.Core.Agents;
18
using Pithos.Interfaces;
19
using System.ServiceModel;
20
using Pithos.Network;
21
using log4net;
22

    
23
namespace Pithos.Core
24
{
25
    [Export(typeof(PithosMonitor))]
26
    public class PithosMonitor:IDisposable
27
    {
28
        private int _blockSize;
29
        private string _blockHash;
30

    
31
        [Import]
32
        public IPithosSettings Settings{get;set;}
33

    
34
        [Import]
35
        public IStatusKeeper StatusKeeper { get; set; }
36

    
37
        [Import]
38
        public IPithosWorkflow Workflow { get; set; }
39

    
40
        public ICloudClient CloudClient { get; set; }
41

    
42
        public IStatusNotification StatusNotification { get; set; }
43

    
44
        [Import]
45
        public FileAgent FileAgent { get; set; }
46
        
47
        [Import]
48
        public WorkflowAgent WorkflowAgent { get; set; }
49
        
50
        [Import]
51
        public NetworkAgent NetworkAgent { get; set; }        
52

    
53
        public string UserName { get; set; }
54
        private string _apiKey;
55
        public string ApiKey
56
        {
57
            get { return _apiKey; }
58
            set
59
            {
60
                _apiKey = value;
61
                if (_accountInfo != null)
62
                    _accountInfo.Token = value;
63
            }
64
        }
65

    
66
        private AccountInfo _accountInfo;
67

    
68

    
69
        private static readonly ILog Log = LogManager.GetLogger(typeof(PithosMonitor));
70

    
71

    
72
        public bool Pause
73
        {
74
            get { return FileAgent.Pause; }
75
            set
76
            {
77
                FileAgent.Pause = value;
78
                if (value)
79
                {
80
                    StatusKeeper.SetPithosStatus(PithosStatus.SyncPaused);
81
                    StatusNotification.NotifyChange("Paused");
82
                }
83
                else
84
                {
85
                    StatusKeeper.SetPithosStatus(PithosStatus.InSynch);
86
                    StatusNotification.NotifyChange("Synchronizing");
87
                }
88
            }
89
        }
90

    
91
        private string _rootPath;
92
        public string RootPath
93
        {
94
            get { return _rootPath; }
95
            set 
96
            {
97
                _rootPath = String.IsNullOrWhiteSpace(value) 
98
                    ? String.Empty 
99
                    : value.ToLower();
100
            }
101
        }
102

    
103

    
104
        CancellationTokenSource _cancellationSource;
105

    
106

    
107
        private bool _started;
108

    
109
        public void Start()
110
        {            
111
            if (String.IsNullOrWhiteSpace(ApiKey))
112
                throw new InvalidOperationException("The ApiKey is empty");
113
            if (String.IsNullOrWhiteSpace(UserName))
114
                throw new InvalidOperationException("The UserName is empty");
115
            if (String.IsNullOrWhiteSpace(AuthenticationUrl))
116
                throw new InvalidOperationException("The Authentication url is empty");
117
            Contract.EndContractBlock();
118

    
119
            //If the account doesn't have a valid path, don't start monitoring but don't throw either
120
            if (String.IsNullOrWhiteSpace(RootPath))
121
                //TODO; Warn user?
122
                return;
123

    
124
            StatusNotification.NotifyChange("Starting");
125
            if (_started)
126
            {
127
                if (!_cancellationSource.IsCancellationRequested)
128
                    return;
129
            }
130
            _cancellationSource = new CancellationTokenSource();
131
            
132

    
133
            CloudClient=new CloudFilesClient(UserName,ApiKey);
134
            var proxyUri = ProxyFromSettings();            
135
            CloudClient.Proxy = proxyUri;
136
            CloudClient.UsePithos = true;
137
            CloudClient.AuthenticationUrl = this.AuthenticationUrl;            
138

    
139
            _accountInfo = CloudClient.Authenticate();            
140
            _accountInfo.SiteUri = AuthenticationUrl;
141
            _accountInfo.AccountPath = RootPath;
142

    
143

    
144
            var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
145
            if (!Directory.Exists(pithosFolder))
146
                Directory.CreateDirectory(pithosFolder);
147
            //Create the cache folder and ensure it is hidden
148
            CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
149

    
150
            var policy=CloudClient.GetAccountPolicies(_accountInfo);
151

    
152
            StatusNotification.NotifyAccount(policy);
153
            EnsurePithosContainers();
154
            
155
            StatusKeeper.BlockHash = _blockHash;
156
            StatusKeeper.BlockSize = _blockSize;
157
            
158
            StatusKeeper.StartProcessing(_cancellationSource.Token);
159
            IndexLocalFiles();
160
            StartWatcherAgent();
161

    
162
            StartNetworkAgent();
163

    
164
            StartWorkflowAgent();
165
            WorkflowAgent.RestartInterruptedFiles(_accountInfo);
166
            _started = true;
167
        }
168

    
169
        private void EnsurePithosContainers()
170
        {
171

    
172
            //Create the two default containers if they are missing
173
            var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
174
            foreach (var container in pithosContainers)
175
            {                
176
                var info=CloudClient.GetContainerInfo(this.UserName, container);
177
                if (info == ContainerInfo.Empty)
178
                {
179
                    CloudClient.CreateContainer(this.UserName, container);
180
                    info = CloudClient.GetContainerInfo(this.UserName, container);
181
                }
182
                _blockSize = info.BlockSize;
183
                _blockHash = info.BlockHash;
184
                _accountInfo.BlockSize = _blockSize;
185
                _accountInfo.BlockHash = _blockHash;
186
            }
187
        }
188

    
189
        public string AuthenticationUrl { get; set; }
190

    
191
        private Uri ProxyFromSettings()
192
        {            
193
            if (Settings.UseManualProxy)
194
            {
195
                var proxyUri = new UriBuilder
196
                                   {
197
                                       Host = Settings.ProxyServer, 
198
                                       Port = Settings.ProxyPort
199
                                   };
200
                if (Settings.ProxyAuthentication)
201
                {
202
                    proxyUri.UserName = Settings.ProxyUsername;
203
                    proxyUri.Password = Settings.ProxyPassword;
204
                }
205
                return proxyUri.Uri;
206
            }
207
            return null;
208
        }
209

    
210
        private void IndexLocalFiles()
211
        {
212
            StatusNotification.NotifyChange("Indexing Local Files",TraceLevel.Info);
213
            using (log4net.ThreadContext.Stacks["Monitor"].Push("Indexing local files"))
214
            {
215
                Log.Info("START");
216
                try
217
                {
218
                    var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
219
                    var directory = new DirectoryInfo(RootPath);
220
                    var files =
221
                        from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
222
                        where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
223
                              !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
224
                        select file;
225
                    StatusKeeper.ProcessExistingFiles(files);
226

    
227
                }
228
                catch (Exception exc)
229
                {
230
                    Log.Error("[ERROR]", exc);
231
                }
232
                finally
233
                {
234
                    Log.Info("[END]");
235
                }
236
            }
237
        }
238

    
239
        
240
  
241

    
242

    
243
        private void StartWorkflowAgent()
244
        {
245

    
246
            bool connected = NetworkListManager.IsConnectedToInternet;
247
            //If we are not connected retry later
248
            if (!connected)
249
            {
250
                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
251
                return;
252
            }
253

    
254
            try
255
            {
256
                WorkflowAgent.StatusNotification = StatusNotification;
257
                WorkflowAgent.Start();                
258
            }
259
            catch (Exception)
260
            {
261
                //Faild to authenticate due to network or account error
262
                //Retry after a while
263
                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
264
            }
265
        }
266

    
267
        internal class LocalFileComparer:EqualityComparer<CloudAction>
268
        {
269
            public override bool Equals(CloudAction x, CloudAction y)
270
            {
271
                if (x.Action != y.Action)
272
                    return false;
273
                if (x.LocalFile != null && y.LocalFile != null && !x.LocalFile.FullName.Equals(y.LocalFile.FullName))
274
                    return false;
275
                if (x.CloudFile != null && y.CloudFile != null )
276
                {
277
                    if (x.CloudFile.Hash == null & y.CloudFile.Hash != null)
278
                        return false;
279
                    if (x.CloudFile.Hash != null & y.CloudFile.Hash == null)
280
                        return false;
281
                    if (x.CloudFile.Hash == null & y.CloudFile.Hash == null)
282
                        return (x.CloudFile.Name == y.CloudFile.Name);
283
                    if (!x.CloudFile.Hash.Equals(y.CloudFile.Hash))
284
                        return false;
285
                }
286
                if (x.CloudFile == null ^ y.CloudFile == null ||
287
                    x.LocalFile == null ^ y.LocalFile == null)
288
                    return false;
289
                return true;
290
            }
291

    
292
            public override int GetHashCode(CloudAction obj)
293
            {
294
                var hash1 = (obj.LocalFile == null) ? int.MaxValue : obj.LocalFile.FullName.GetHashCode();
295
                var hash2 = (obj.CloudFile == null) ? int.MaxValue : (obj.CloudFile.Hash ?? obj.CloudFile.Name??"").GetHashCode();
296
                var hash3 = obj.Action.GetHashCode();
297
                return hash1 ^ hash2 & hash3;
298
            }
299
        }        
300

    
301
        private Timer timer;
302

    
303
        private void StartNetworkAgent()
304
        {
305

    
306
            NetworkAgent.AddAccount(_accountInfo);
307

    
308
            NetworkAgent.StatusNotification = StatusNotification;
309
                        
310
            NetworkAgent.Start();
311

    
312
            NetworkAgent.ProcessRemoteFiles();
313
        }
314

    
315
        //Make sure a hidden cache folder exists to store partial downloads
316
        private static string CreateHiddenFolder(string rootPath, string folderName)
317
        {
318
            if (String.IsNullOrWhiteSpace(rootPath))
319
                throw new ArgumentNullException("rootPath");
320
            if (!Path.IsPathRooted(rootPath))
321
                throw new ArgumentException("rootPath");
322
            if (String.IsNullOrWhiteSpace(folderName))
323
                throw new ArgumentNullException("folderName");
324
            Contract.EndContractBlock();
325

    
326
            var folder = Path.Combine(rootPath, folderName);
327
            if (!Directory.Exists(folder))
328
            {
329
                var info = Directory.CreateDirectory(folder);
330
                info.Attributes |= FileAttributes.Hidden;
331

    
332
                Log.InfoFormat("Created cache Folder: {0}", folder);
333
            }
334
            else
335
            {
336
                var info = new DirectoryInfo(folder);
337
                if ((info.Attributes & FileAttributes.Hidden) == 0)
338
                {
339
                    info.Attributes |= FileAttributes.Hidden;
340
                    Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
341
                }                                
342
            }
343
            return folder;
344
        }
345

    
346
       
347

    
348

    
349
        private void StartWatcherAgent()
350
        {
351
            AgentLocator<FileAgent>.Register(FileAgent,RootPath);
352

    
353
            FileAgent.StatusKeeper = StatusKeeper;
354
            FileAgent.Workflow = Workflow;
355
            FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
356
            FileAgent.Start(_accountInfo, RootPath);
357
        }
358

    
359
        public void Stop()
360
        {
361
            AgentLocator<FileAgent>.Remove(RootPath);
362

    
363
            if (FileAgent!=null)
364
                FileAgent.Stop();
365
            FileAgent = null;
366
            if (timer != null)
367
                timer.Dispose();
368
            timer = null;            
369
        }
370

    
371

    
372
        ~PithosMonitor()
373
        {
374
            Dispose(false);
375
        }
376

    
377
        public void Dispose()
378
        {
379
            Dispose(true);
380
            GC.SuppressFinalize(this);
381
        }
382

    
383
        protected virtual void Dispose(bool disposing)
384
        {
385
            if (disposing)
386
            {
387
                Stop();
388
            }
389
        }
390

    
391

    
392
        public void MoveFileStates(string oldPath, string newPath)
393
        {
394
            if (String.IsNullOrWhiteSpace(oldPath))
395
                throw new ArgumentNullException("oldPath");
396
            if (!Path.IsPathRooted(oldPath))
397
                throw new ArgumentException("oldPath must be an absolute path","oldPath");
398
            if (string.IsNullOrWhiteSpace(newPath))
399
                throw new ArgumentNullException("newPath");
400
            if (!Path.IsPathRooted(newPath))
401
                throw new ArgumentException("newPath must be an absolute path","newPath");
402
            Contract.EndContractBlock();
403

    
404
            StatusKeeper.ChangeRoots(oldPath, newPath);
405
        }
406

    
407
        public void AddSelectivePaths(string[] added)
408
        {
409
           /* FileAgent.SelectivePaths.AddRange(added);
410
            NetworkAgent.SyncPaths(added);*/
411
        }
412

    
413
        public void RemoveSelectivePaths(string[] removed)
414
        {
415
            FileAgent.SelectivePaths.RemoveAll(removed.Contains);
416
            foreach (var removedPath in removed.Where(Directory.Exists))
417
            {
418
                Directory.Delete(removedPath,true);
419
            }
420
        }
421

    
422
        public IEnumerable<string> GetRootFolders()
423
        {
424
            var dirs = from container in CloudClient.ListContainers(UserName)
425
                       from dir in CloudClient.ListObjects(UserName, container.Name, "")
426
                       select dir.Name;
427
            return dirs;
428
        }
429

    
430
        public ObjectInfo GetObjectInfo(string filePath)
431
        {
432
            if (String.IsNullOrWhiteSpace(filePath))
433
                throw new ArgumentNullException("filePath");
434
            Contract.EndContractBlock();
435

    
436
            var file=new FileInfo(filePath);
437
            string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
438
            var relativePath = file.AsRelativeTo(RootPath);
439
            
440
            string accountName,container;
441
            
442
            var parts=relativePath.Split('\\');
443

    
444
            var accountInfo = _accountInfo;
445
            if (relativePath.StartsWith(FolderConstants.OthersFolder))
446
            {                
447
                accountName = parts[1];
448
                container = parts[2];
449
                relativeUrl = String.Join("/", parts.Splice(3));
450
                //Create the root URL for the target account
451
                var oldName = UserName;
452
                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
453
                var nameIndex=absoluteUri.IndexOf(oldName);
454
                var root=absoluteUri.Substring(0, nameIndex);
455

    
456
                accountInfo = new AccountInfo
457
                {
458
                    UserName = accountName,
459
                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
460
                    StorageUri = new Uri(root + accountName),
461
                    BlockHash=accountInfo.BlockHash,
462
                    BlockSize=accountInfo.BlockSize,
463
                    Token=accountInfo.Token
464
                };
465
            }
466
            else
467
            {
468
                accountName = this.UserName;
469
                container = parts[0];
470
                relativeUrl = String.Join("/", parts.Splice(1));
471
            }
472
            
473
            var client = new CloudFilesClient(accountInfo);
474
            var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
475
            return objectInfo;
476
        }
477
        
478
        public Task<ContainerInfo> GetContainerInfo(string filePath)
479
        {
480
            if (String.IsNullOrWhiteSpace(filePath))
481
                throw new ArgumentNullException("filePath");
482
            Contract.EndContractBlock();
483

    
484
            var file=new FileInfo(filePath);
485
            var relativePath = file.AsRelativeTo(RootPath);
486
            
487
            string accountName,container;
488
            
489
            var parts=relativePath.Split('\\');
490

    
491
            var accountInfo = _accountInfo;
492
            if (relativePath.StartsWith(FolderConstants.OthersFolder))
493
            {                
494
                accountName = parts[1];
495
                container = parts[2];                
496
                //Create the root URL for the target account
497
                var oldName = UserName;
498
                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
499
                var nameIndex=absoluteUri.IndexOf(oldName);
500
                var root=absoluteUri.Substring(0, nameIndex);
501

    
502
                accountInfo = new AccountInfo
503
                {
504
                    UserName = accountName,
505
                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
506
                    StorageUri = new Uri(root + accountName),
507
                    BlockHash=accountInfo.BlockHash,
508
                    BlockSize=accountInfo.BlockSize,
509
                    Token=accountInfo.Token
510
                };
511
            }
512
            else
513
            {
514
                accountName = UserName;
515
                container = parts[0];                
516
            }
517

    
518
            return Task.Factory.StartNew(() =>
519
            {
520
                var client = new CloudFilesClient(accountInfo);
521
                var containerInfo = client.GetContainerInfo(accountName, container);
522
                return containerInfo;
523
            });
524
        }
525
    }
526

    
527

    
528
    public interface IStatusNotification
529
    {        
530
        void NotifyChange(string status,TraceLevel level=TraceLevel.Info);
531
        void NotifyChangedFile(string filePath);
532
        void NotifyAccount(AccountInfo policy);
533
    }
534
}