Statistics
| Branch: | Revision:

root / trunk / Pithos.Core / PithosMonitor.cs @ 7b0a5fec

History | View | Annotate | Download (16.4 kB)

1
using System;
2
using System.Collections.Concurrent;
3
using System.Collections.Generic;
4
using System.ComponentModel.Composition;
5
using System.Diagnostics;
6
using System.Diagnostics.Contracts;
7
using System.IO;
8
using System.Linq;
9
using System.Net.NetworkInformation;
10
using System.Security.Cryptography;
11
using System.ServiceModel.Description;
12
using System.Text;
13
using System.Threading;
14
using System.Threading.Tasks;
15
using Castle.ActiveRecord.Queries;
16
using Microsoft.WindowsAPICodePack.Net;
17
using Pithos.Core.Agents;
18
using Pithos.Interfaces;
19
using System.ServiceModel;
20
using Pithos.Network;
21
using log4net;
22

    
23
namespace Pithos.Core
24
{
25
    [Export(typeof(PithosMonitor))]
26
    public class PithosMonitor:IDisposable
27
    {
28
        private int _blockSize;
29
        private string _blockHash;
30

    
31
        [Import]
32
        public IPithosSettings Settings{get;set;}
33

    
34
        [Import]
35
        public IStatusKeeper StatusKeeper { get; set; }
36

    
37
        [Import]
38
        public IPithosWorkflow Workflow { get; set; }
39

    
40
        public ICloudClient CloudClient { get; set; }
41

    
42
        public IStatusNotification StatusNotification { get; set; }
43

    
44
        [Import]
45
        public FileAgent FileAgent { get; set; }
46
        
47
        [Import]
48
        public WorkflowAgent WorkflowAgent { get; set; }
49
        
50
        [Import]
51
        public NetworkAgent NetworkAgent { get; set; }        
52

    
53
        public string UserName { get; set; }
54
        public string ApiKey { get; set; }
55

    
56
        private AccountInfo _accountInfo;
57

    
58

    
59
        private static readonly ILog Log = LogManager.GetLogger(typeof(PithosMonitor));
60

    
61

    
62
        public bool Pause
63
        {
64
            get { return FileAgent.Pause; }
65
            set
66
            {
67
                FileAgent.Pause = value;
68
                if (value)
69
                {
70
                    StatusKeeper.SetPithosStatus(PithosStatus.SyncPaused);
71
                    StatusNotification.NotifyChange("Paused");
72
                }
73
                else
74
                {
75
                    StatusKeeper.SetPithosStatus(PithosStatus.InSynch);
76
                    StatusNotification.NotifyChange("Synchronizing");
77
                }
78
            }
79
        }
80

    
81
        private string _rootPath;
82
        public string RootPath
83
        {
84
            get { return _rootPath; }
85
            set 
86
            {
87
                _rootPath = String.IsNullOrWhiteSpace(value) 
88
                    ? String.Empty 
89
                    : value.ToLower();
90
            }
91
        }
92

    
93

    
94
        CancellationTokenSource _cancellationSource;
95

    
96

    
97
        private bool _started;
98

    
99
        public void Start()
100
        {            
101
            if (String.IsNullOrWhiteSpace(ApiKey))
102
                throw new InvalidOperationException("The ApiKey is empty");
103
            if (String.IsNullOrWhiteSpace(UserName))
104
                throw new InvalidOperationException("The UserName is empty");
105
            if (String.IsNullOrWhiteSpace(AuthenticationUrl))
106
                throw new InvalidOperationException("The Authentication url is empty");
107
            Contract.EndContractBlock();
108

    
109
            StatusNotification.NotifyChange("Starting");
110
            if (_started)
111
            {
112
                if (!_cancellationSource.IsCancellationRequested)
113
                    return;
114
            }
115
            _cancellationSource = new CancellationTokenSource();
116
            _started = true;
117

    
118
            CloudClient=new CloudFilesClient(UserName,ApiKey);
119
            var proxyUri = ProxyFromSettings();            
120
            CloudClient.Proxy = proxyUri;
121
            CloudClient.UsePithos = this.UsePithos;
122
            CloudClient.AuthenticationUrl = this.AuthenticationUrl;            
123

    
124
            _accountInfo = CloudClient.Authenticate();
125
            _accountInfo.AccountPath = RootPath;
126

    
127

    
128
            var pithosFolder = Path.Combine(RootPath, FolderConstants.PithosContainer);
129
            if (!Directory.Exists(pithosFolder))
130
                Directory.CreateDirectory(pithosFolder);
131
            //Create the cache folder and ensure it is hidden
132
            CreateHiddenFolder(RootPath, FolderConstants.CacheFolder);
133

    
134
            var policy=CloudClient.GetAccountPolicies(_accountInfo);
135

    
136
            StatusNotification.NotifyAccount(policy);
137
            EnsurePithosContainers();
138
            
139
            StatusKeeper.BlockHash = _blockHash;
140
            StatusKeeper.BlockSize = _blockSize;
141
            
142
            StatusKeeper.StartProcessing(_cancellationSource.Token);
143
            IndexLocalFiles();
144
            StartWatcherAgent();
145

    
146
            StartNetworkAgent();
147

    
148
            StartWorkflowAgent();
149
            WorkflowAgent.RestartInterruptedFiles(_accountInfo);            
150
        }
151

    
152
        private void EnsurePithosContainers()
153
        {
154

    
155
            //Create the two default containers if they are missing
156
            var pithosContainers = new List<string>{ FolderConstants.TrashContainer,FolderConstants.PithosContainer};
157
            foreach (var container in pithosContainers)
158
            {                
159
                var info=CloudClient.GetContainerInfo(this.UserName, container);
160
                if (info == ContainerInfo.Empty)
161
                {
162
                    CloudClient.CreateContainer(this.UserName, container);
163
                    info = CloudClient.GetContainerInfo(this.UserName, container);
164
                }
165
                _blockSize = info.BlockSize;
166
                _blockHash = info.BlockHash;
167
                _accountInfo.BlockSize = _blockSize;
168
                _accountInfo.BlockHash = _blockHash;
169
            }
170
        }
171

    
172
        public string AuthenticationUrl { get; set; }
173

    
174
        private Uri ProxyFromSettings()
175
        {            
176
            if (Settings.UseManualProxy)
177
            {
178
                var proxyUri = new UriBuilder
179
                                   {
180
                                       Host = Settings.ProxyServer, 
181
                                       Port = Settings.ProxyPort
182
                                   };
183
                if (Settings.ProxyAuthentication)
184
                {
185
                    proxyUri.UserName = Settings.ProxyUsername;
186
                    proxyUri.Password = Settings.ProxyPassword;
187
                }
188
                return proxyUri.Uri;
189
            }
190
            return null;
191
        }
192

    
193
        private void IndexLocalFiles()
194
        {
195
            StatusNotification.NotifyChange("Indexing Local Files",TraceLevel.Info);
196
            using (log4net.ThreadContext.Stacks["Monitor"].Push("Indexing local files"))
197
            {
198
                Log.Info("START");
199
                try
200
                {
201
                    var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
202
                    var directory = new DirectoryInfo(RootPath);
203
                    var files =
204
                        from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
205
                        where !file.FullName.StartsWith(cachePath, StringComparison.InvariantCultureIgnoreCase) &&
206
                              !file.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)
207
                        select file;
208
                    StatusKeeper.ProcessExistingFiles(files);
209

    
210
                }
211
                catch (Exception exc)
212
                {
213
                    Log.Error("[ERROR]", exc);
214
                }
215
                finally
216
                {
217
                    Log.Info("[END]");
218
                }
219
            }
220
        }
221

    
222
        
223
  
224

    
225

    
226
        private void StartWorkflowAgent()
227
        {
228

    
229
            bool connected = NetworkListManager.IsConnectedToInternet;
230
            //If we are not connected retry later
231
            if (!connected)
232
            {
233
                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
234
                return;
235
            }
236

    
237
            try
238
            {
239
                WorkflowAgent.StatusNotification = StatusNotification;
240
                WorkflowAgent.Start();                
241
            }
242
            catch (Exception)
243
            {
244
                //Faild to authenticate due to network or account error
245
                //Retry after a while
246
                Task.Factory.StartNewDelayed(10000, StartWorkflowAgent);
247
            }
248
        }
249

    
250
        public bool UsePithos { get; set; }
251

    
252

    
253
        internal class LocalFileComparer:EqualityComparer<CloudAction>
254
        {
255
            public override bool Equals(CloudAction x, CloudAction y)
256
            {
257
                if (x.Action != y.Action)
258
                    return false;
259
                if (x.LocalFile != null && y.LocalFile != null && !x.LocalFile.FullName.Equals(y.LocalFile.FullName))
260
                    return false;
261
                if (x.CloudFile != null && y.CloudFile != null )
262
                {
263
                    if (x.CloudFile.Hash == null & y.CloudFile.Hash != null)
264
                        return false;
265
                    if (x.CloudFile.Hash != null & y.CloudFile.Hash == null)
266
                        return false;
267
                    if (x.CloudFile.Hash == null & y.CloudFile.Hash == null)
268
                        return (x.CloudFile.Name == y.CloudFile.Name);
269
                    if (!x.CloudFile.Hash.Equals(y.CloudFile.Hash))
270
                        return false;
271
                }
272
                if (x.CloudFile == null ^ y.CloudFile == null ||
273
                    x.LocalFile == null ^ y.LocalFile == null)
274
                    return false;
275
                return true;
276
            }
277

    
278
            public override int GetHashCode(CloudAction obj)
279
            {
280
                var hash1 = (obj.LocalFile == null) ? int.MaxValue : obj.LocalFile.FullName.GetHashCode();
281
                var hash2 = (obj.CloudFile == null) ? int.MaxValue : (obj.CloudFile.Hash ?? obj.CloudFile.Name??"").GetHashCode();
282
                var hash3 = obj.Action.GetHashCode();
283
                return hash1 ^ hash2 & hash3;
284
            }
285
        }        
286

    
287
        private Timer timer;
288

    
289
        private void StartNetworkAgent()
290
        {
291

    
292
            NetworkAgent.AddAccount(_accountInfo);
293

    
294
            NetworkAgent.StatusNotification = StatusNotification;
295
                        
296
            NetworkAgent.Start();
297

    
298
            NetworkAgent.ProcessRemoteFiles();
299
        }
300

    
301
        //Make sure a hidden cache folder exists to store partial downloads
302
        private static string CreateHiddenFolder(string rootPath, string folderName)
303
        {
304
            if (String.IsNullOrWhiteSpace(rootPath))
305
                throw new ArgumentNullException("rootPath");
306
            if (!Path.IsPathRooted(rootPath))
307
                throw new ArgumentException("rootPath");
308
            if (String.IsNullOrWhiteSpace(folderName))
309
                throw new ArgumentNullException("folderName");
310
            Contract.EndContractBlock();
311

    
312
            var folder = Path.Combine(rootPath, folderName);
313
            if (!Directory.Exists(folder))
314
            {
315
                var info = Directory.CreateDirectory(folder);
316
                info.Attributes |= FileAttributes.Hidden;
317

    
318
                Log.InfoFormat("Created cache Folder: {0}", folder);
319
            }
320
            else
321
            {
322
                var info = new DirectoryInfo(folder);
323
                if ((info.Attributes & FileAttributes.Hidden) == 0)
324
                {
325
                    info.Attributes |= FileAttributes.Hidden;
326
                    Log.InfoFormat("Reset cache folder to hidden: {0}", folder);
327
                }                                
328
            }
329
            return folder;
330
        }
331

    
332
       
333

    
334

    
335
        private void StartWatcherAgent()
336
        {
337
            AgentLocator<FileAgent>.Register(FileAgent,RootPath);
338

    
339
            FileAgent.StatusKeeper = StatusKeeper;
340
            FileAgent.Workflow = Workflow;
341
            FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
342
            FileAgent.Start(_accountInfo, RootPath);
343
        }
344

    
345
        public void Stop()
346
        {
347
            AgentLocator<FileAgent>.Remove(RootPath);
348

    
349
            if (FileAgent!=null)
350
                FileAgent.Stop();
351
            FileAgent = null;
352
            if (timer != null)
353
                timer.Dispose();
354
            timer = null;            
355
        }
356

    
357

    
358
        ~PithosMonitor()
359
        {
360
            Dispose(false);
361
        }
362

    
363
        public void Dispose()
364
        {
365
            Dispose(true);
366
            GC.SuppressFinalize(this);
367
        }
368

    
369
        protected virtual void Dispose(bool disposing)
370
        {
371
            if (disposing)
372
            {
373
                Stop();
374
            }
375
        }
376

    
377

    
378
        public void MoveFileStates(string oldPath, string newPath)
379
        {
380
            if (String.IsNullOrWhiteSpace(oldPath))
381
                throw new ArgumentNullException("oldPath");
382
            if (!Path.IsPathRooted(oldPath))
383
                throw new ArgumentException("oldPath must be an absolute path","oldPath");
384
            if (string.IsNullOrWhiteSpace(newPath))
385
                throw new ArgumentNullException("newPath");
386
            if (!Path.IsPathRooted(newPath))
387
                throw new ArgumentException("newPath must be an absolute path","newPath");
388
            Contract.EndContractBlock();
389

    
390
            StatusKeeper.ChangeRoots(oldPath, newPath);
391
        }
392

    
393
        public void AddSelectivePaths(string[] added)
394
        {
395
           /* FileAgent.SelectivePaths.AddRange(added);
396
            NetworkAgent.SyncPaths(added);*/
397
        }
398

    
399
        public void RemoveSelectivePaths(string[] removed)
400
        {
401
            FileAgent.SelectivePaths.RemoveAll(removed.Contains);
402
            foreach (var removedPath in removed.Where(Directory.Exists))
403
            {
404
                Directory.Delete(removedPath,true);
405
            }
406
        }
407

    
408
        public IEnumerable<string> GetRootFolders()
409
        {
410
            var dirs = from container in CloudClient.ListContainers(UserName)
411
                       from dir in CloudClient.ListObjects(UserName, container.Name, "")
412
                       select dir.Name;
413
            return dirs;
414
        }
415

    
416
        public ObjectInfo GetObjectInfo(string filePath)
417
        {
418
            if (String.IsNullOrWhiteSpace(filePath))
419
                throw new ArgumentNullException("filePath");
420
            Contract.EndContractBlock();
421

    
422
            var file=new FileInfo(filePath);
423
            string relativeUrl;//=file.AsRelativeUrlTo(this.RootPath);
424
            var relativePath = file.AsRelativeTo(RootPath);
425
            
426
            string accountName,container;
427
            
428
            var parts=relativePath.Split('\\');
429

    
430
            var accountInfo = _accountInfo;
431
            if (relativePath.StartsWith(FolderConstants.OthersFolder))
432
            {                
433
                accountName = parts[1];
434
                container = parts[2];
435
                relativeUrl = String.Join("/", parts.Splice(3));
436
                //Create the root URL for the target account
437
                var oldName = UserName;
438
                var absoluteUri =  _accountInfo.StorageUri.AbsoluteUri;
439
                var nameIndex=absoluteUri.IndexOf(oldName);
440
                var root=absoluteUri.Substring(0, nameIndex);
441

    
442
                accountInfo = new AccountInfo
443
                {
444
                    UserName = accountName,
445
                    AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
446
                    StorageUri = new Uri(root + accountName),
447
                    BlockHash=accountInfo.BlockHash,
448
                    BlockSize=accountInfo.BlockSize,
449
                    Token=accountInfo.Token
450
                };
451
            }
452
            else
453
            {
454
                accountName = this.UserName;
455
                container = parts[0];
456
                relativeUrl = String.Join("/", parts.Splice(1));
457
            }
458

    
459
            /*return new ObjectInfo
460
                       {
461
                           Account = accountName,
462
                           Container = container,
463
                           Name = relativeUrl,
464
                           Content_Type="application/pdf",
465
                           Bytes = 123456,
466
                           Hash = "8902372934sjhshjfsdjkf223894sdgh",
467
                           Last_Modified = DateTime.Today.AddDays(-1),
468
                           ModifiedBy = accountName,
469
                           Version = 12312455,
470
                           VersionTimestamp = DateTime.Today.AddDays(-1)
471
                       };*/
472

    
473
            var client = new CloudFilesClient(accountInfo);
474
            var objectInfo=client.GetObjectInfo(accountName, container, relativeUrl);
475
            return objectInfo;
476
        }
477
    }
478

    
479

    
480
    public interface IStatusNotification
481
    {        
482
        void NotifyChange(string status,TraceLevel level=TraceLevel.Info);
483
        void NotifyChangedFile(string filePath);
484
        void NotifyAccount(AccountInfo policy);
485
    }
486
}