Revision c28a075a

b/trunk/Pithos.Client.WPF/PithosAccount.cs
52 52
            Log.InfoFormat("[RETRIEVE] Listening at {0}", listenerUrl);
53 53

  
54 54
            listener.Start();
55
            
56
            var startListening = Task.Factory.FromAsync<HttpListenerContext>(listener.BeginGetContext, listener.EndGetContext, null)
57
                .WithTimeout(TimeSpan.FromMinutes(5));
55 58

  
56
            var task = Task.Factory.FromAsync<HttpListenerContext>(listener.BeginGetContext, listener.EndGetContext, null)
57
                .WithTimeout(TimeSpan.FromMinutes(1))
58
                .ContinueWith(tc =>
59
            var receiveCredentials=startListening.ContinueWith(tc =>
59 60
                {
60 61
                    try
61 62
                    {
......
96 97
            Log.InfoFormat("[RETRIEVE] Open Browser at {0}", retrieveUri);
97 98
            Process.Start(retrieveUri.ToString());
98 99

  
99
            return task;
100
            return receiveCredentials;
100 101
        }
101 102

  
102 103
        private static void Respond(HttpListenerContext context)
b/trunk/Pithos.Client.WPF/ShellViewModel.cs
1 1
using System.Collections.Concurrent;
2 2
using System.ComponentModel.Composition;
3 3
using System.Diagnostics;
4
using System.Diagnostics.Contracts;
4 5
using System.IO;
6
using System.Net;
5 7
using System.Runtime.InteropServices;
6 8
using System.ServiceModel;
7 9
using System.ServiceModel.Description;
......
52 54
            _statusChecker = statusChecker;
53 55
            _events = events;            
54 56
            Settings = settings;
55
            
56
            
57

  
57
                                   
58 58
            UsageMessage = "Using 15% of 50 GB";
59 59
            StatusMessage = "In Synch";
60 60

  
......
77 77
            if (_monitors.TryGetValue(accountName,out monitor))
78 78
            {
79 79
                //If the account is active
80
                if (account.IsActive)
80
                if (account.IsActive)                    
81 81
                    //Start the monitor. It's OK to start an already started monitor,
82 82
                    //it will just ignore the call
83 83
                    monitor.Start();
......
268 268

  
269 269

  
270 270
       
271
        private Task StartMonitor(PithosMonitor monitor)
271
        private Task StartMonitor(PithosMonitor monitor,int retries=0)
272 272
        {
273 273
            return Task.Factory.StartNew(() =>
274 274
            {
......
277 277
                    try
278 278
                    {
279 279
                        Log.InfoFormat("Start Monitoring {0}", monitor.UserName);
280

  
280 281
                        monitor.Start();
281 282
                    }
283
                    catch (WebException exc)
284
                    {
285
                        if (AbandonRetry(monitor, retries))
286
                            return;
287

  
288
                        if (IsUnauthorized(exc))
289
                        {
290
                            var message = String.Format("API Key Expired for {0}. Starting Renewal",monitor.UserName);                            
291
                            Log.Error(message,exc);
292
                            TryAuthorize(monitor,retries);
293
                        }
294
                        else
295
                        {
296
                            TryLater(monitor, exc,retries);
297
                        }
298
                    }
282 299
                    catch (Exception exc)
283 300
                    {
284
                        var message =
285
                            String.Format("An exception occured. Can't start monitoring\nWill retry in 10 seconds");
286
                        Task.Factory.StartNewDelayed(10000, () => StartMonitor(monitor));
287
                        _events.Publish(new Notification {Title = "Error", Message = message, Level = TraceLevel.Error});
288
                        Log.Error(message, exc);
301
                        if (AbandonRetry(monitor, retries)) 
302
                            return;
303

  
304
                        TryLater(monitor,exc,retries);
289 305
                    }
290 306
                }
291 307
            });
292 308
        }
293 309

  
310
        private bool AbandonRetry(PithosMonitor monitor, int retries)
311
        {
312
            if (retries > 1)
313
            {
314
                var message = String.Format("Monitoring of account {0} has failed too many times. Will not retry",
315
                                            monitor.UserName);
316
                _events.Publish(new Notification
317
                                    {Title = "Account monitoring failed", Message = message, Level = TraceLevel.Error});
318
                return true;
319
            }
320
            return false;
321
        }
322

  
323

  
324
        private Task TryAuthorize(PithosMonitor monitor,int retries)
325
        {
326
            _events.Publish(new Notification { Title = "Authorization failed", Message = "Your API Key has probably expired. You will be directed to a page where you can renew it", Level = TraceLevel.Error });
327

  
328
            var authorize= PithosAccount.RetrieveCredentialsAsync(Settings.PithosSite);
329

  
330
            return authorize.ContinueWith(t =>
331
            {
332
                if (t.IsFaulted)
333
                {                    
334
                    string message = String.Format("API Key retrieval for {0} failed", monitor.UserName);
335
                    Log.Error(message,t.Exception.InnerException);
336
                    _events.Publish(new Notification { Title = "Authorization failed", Message = message, Level = TraceLevel.Error });
337
                    return;
338
                }
339
                var credentials = t.Result;
340
                var account =Settings.Accounts.FirstOrDefault(act => act.AccountName == credentials.UserName);
341
                account.ApiKey = credentials.Password;
342
                monitor.ApiKey = credentials.Password;
343
                Task.Factory.StartNewDelayed(10000, () => StartMonitor(monitor,retries+1));
344
            });
345
        }
346

  
347
        private static bool IsUnauthorized(WebException exc)
348
        {
349
            if (exc==null)
350
                throw new ArgumentNullException("exc");
351
            Contract.EndContractBlock();
352

  
353
            var response = exc.Response as HttpWebResponse;
354
            if (response == null)
355
                return false;
356
            return (response.StatusCode == HttpStatusCode.Unauthorized);
357
        }
358

  
359
        private void TryLater(PithosMonitor monitor, Exception exc,int retries)
360
        {
361
            var message = String.Format("An exception occured. Can't start monitoring\nWill retry in 10 seconds");
362
            Task.Factory.StartNewDelayed(10000, () => StartMonitor(monitor,retries+1));
363
            _events.Publish(new Notification
364
                                {Title = "Error", Message = message, Level = TraceLevel.Error});
365
            Log.Error(message, exc);
366
        }
367

  
294 368

  
295 369
        public void NotifyChange(string status, TraceLevel level=TraceLevel.Info)
296 370
        {
b/trunk/Pithos.Core.Test/NetworkAgentTest.cs
87 87
                File.Delete(filePath);
88 88

  
89 89
            var newHash = client.GetHashMap(null, FolderConstants.PithosContainer, fileName).Result;
90
            agent.DownloadWithBlocks(client, account, FolderConstants.PithosContainer, new Uri(fileName, UriKind.Relative), filePath, newHash)
90
            agent.DownloadWithBlocks(accountInfo, client, account, FolderConstants.PithosContainer, new Uri(fileName, UriKind.Relative), filePath, newHash)
91 91
                .Wait();
92 92

  
93 93
            Assert.IsTrue(File.Exists(filePath));
b/trunk/Pithos.Core/Agents/AgentLocator.cs
1
using System;
2
using System.Collections.Concurrent;
3
using System.Collections.Generic;
4
using System.Linq;
5
using System.Text;
6

  
7
namespace Pithos.Core.Agents
8
{
9
    static class AgentLocator<T> where T:class
10
    {
11
        static ConcurrentDictionary<string, WeakReference> _agents = new ConcurrentDictionary<string, WeakReference>();
12
        public static void Register(T agent,string key)
13
        {            
14
            _agents[key] = new WeakReference(agent);
15
        }
16

  
17
        public static T Get(string key)
18
        {            
19
            return _agents[key].Target as T;
20
        }
21

  
22
        public static bool TryGet(string key, out T value)
23
        {
24
            WeakReference target;
25
            var exists = _agents.TryGetValue(key, out target);            
26
            value = target.Target as T;
27
            return exists;
28
        }
29

  
30
        public static void Remove(string key)
31
        {
32
            WeakReference target;
33
            _agents.TryRemove(key, out target);
34
        }
35
    }
36
}
b/trunk/Pithos.Core/Agents/FileAgent.cs
14 14

  
15 15
namespace Pithos.Core.Agents
16 16
{
17
    [Export,PartCreationPolicy(CreationPolicy.NonShared)]
17
    [Export]
18 18
    public class FileAgent
19 19
    {
20 20
        Agent<WorkflowState> _agent;
b/trunk/Pithos.Core/Agents/NetworkAgent.cs
24 24
        public IStatusKeeper StatusKeeper { get; set; }
25 25
        
26 26
        public IStatusNotification StatusNotification { get; set; }
27
/*
27 28
        [Import]
28 29
        public FileAgent FileAgent {get;set;}
30
*/
29 31

  
30 32
       /* public int BlockSize { get; set; }
31 33
        public string BlockHash { get; set; }*/
......
320 322
                              select ProcessAccountFiles(accountInfo, since);
321 323
                    var process=Task.Factory.Iterate(tasks);
322 324

  
323
                    return process.ContinueWith(t=>
324
                        ProcessRemoteFiles(nextSince));
325
                    return process.ContinueWith(t =>
326
                    {
327
                        if (t.IsFaulted)
328
                        {
329
                            Log.Error("Error while processing accounts");
330
                            t.Exception.Handle(exc=>
331
                                                   {
332
                                                       Log.Error("Details:", exc);
333
                                                       return true;
334
                                                   });                            
335
                        }
336
                        ProcessRemoteFiles(nextSince);
337
                    });
325 338
                }
326 339
            });            
327 340
        }
......
429 442
            if (remote==null)
430 443
                throw new ArgumentNullException();
431 444
            Contract.EndContractBlock();
445
            var fileAgent = GetFileAgent(accountInfo);
432 446

  
433 447
            //In order to avoid multiple iterations over the files, we iterate only once
434 448
            //over the remote files
......
436 450
            {
437 451
                var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);
438 452
                //and remove any matching objects from the list, adding them to the commonObjects list
439
                if (FileAgent.Exists(relativePath))
453
                
454
                if (fileAgent.Exists(relativePath))
440 455
                {
441
                    var localFile = FileAgent.GetFileInfo(relativePath);
456
                    var localFile = fileAgent.GetFileInfo(relativePath);
442 457
                    var state = FileState.FindByFilePath(localFile.FullName);
443 458
                    //Common files should be checked on a per-case basis to detect differences, which is newer
444 459

  
......
461 476
            }            
462 477
        }
463 478

  
479
        private static FileAgent GetFileAgent(AccountInfo accountInfo)
480
        {
481
            return AgentLocator<FileAgent>.Get(accountInfo.AccountPath);
482
        }
483

  
464 484
        private void ProcessDeletedFiles(AccountInfo accountInfo,IEnumerable<ObjectInfo> trashObjects)
465 485
        {
486
            var fileAgent = GetFileAgent(accountInfo);
466 487
            foreach (var trashObject in trashObjects)
467 488
            {
468 489
                var relativePath = trashObject.RelativeUrlToFilePath(accountInfo.UserName);
469 490
                //and remove any matching objects from the list, adding them to the commonObjects list
470
                FileAgent.Delete(relativePath);                                
491
                fileAgent.Delete(relativePath);                                
471 492
            }
472 493
        }
473 494

  
......
514 535
            if (Path.IsPathRooted(fileName))
515 536
                throw new ArgumentException("The fileName should not be rooted","fileName");
516 537
            Contract.EndContractBlock();
538
            
539
            var fileAgent = GetFileAgent(accountInfo);
517 540

  
518 541
            using ( log4net.ThreadContext.Stacks["DeleteCloudFile"].Push("Delete"))
519 542
            {
520
                var info = FileAgent.GetFileInfo(fileName);
543
                var info = fileAgent.GetFileInfo(fileName);
521 544
                var fullPath = info.FullName.ToLower();
522 545
                this.StatusKeeper.SetFileOverlayStatus(fullPath, FileOverlayStatus.Modified);
523 546

  
......
586 609
                //If it's a small file
587 610
                var downloadTask=(serverHash.Hashes.Count == 1 )
588 611
                    //Download it in one go
589
                    ? DownloadEntireFile(client, account, container, relativeUrl, localPath) 
612
                    ? DownloadEntireFile(accountInfo,client, account, container, relativeUrl, localPath) 
590 613
                    //Otherwise download it block by block
591
                    : DownloadWithBlocks(client, account, container, relativeUrl, localPath, serverHash);
614
                    : DownloadWithBlocks(accountInfo,client, account, container, relativeUrl, localPath, serverHash);
592 615

  
593 616
                yield return downloadTask;
594 617

  
......
605 628
        }
606 629

  
607 630
        //Download a small file with a single GET operation
608
        private Task DownloadEntireFile(CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath)
631
        private Task DownloadEntireFile(AccountInfo accountInfo, CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath)
609 632
        {
610 633
            if (client == null)
611 634
                throw new ArgumentNullException("client");
......
621 644
                throw new ArgumentException("The localPath must be rooted", "localPath");
622 645
            Contract.EndContractBlock();
623 646

  
647
            var fileAgent = GetFileAgent(accountInfo);
624 648
            //Calculate the relative file path for the new file
625 649
            var relativePath = relativeUrl.RelativeUriToFilePath();
626 650
            //The file will be stored in a temporary location while downloading with an extension .download
627
            var tempPath = Path.Combine(FileAgent.FragmentsPath, relativePath + ".download");
651
            var tempPath = Path.Combine(fileAgent.FragmentsPath, relativePath + ".download");
628 652
            //Make sure the target folder exists. DownloadFileTask will not create the folder
629 653
            var directoryPath = Path.GetDirectoryName(tempPath);
630 654
            if (!Directory.Exists(directoryPath))
......
644 668
        }
645 669

  
646 670
        //Download a file asynchronously using blocks
647
        public Task DownloadWithBlocks(CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath, TreeHash serverHash)
671
        public Task DownloadWithBlocks(AccountInfo accountInfo, CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath, TreeHash serverHash)
648 672
        {
649 673
            if (client == null)
650 674
                throw new ArgumentNullException("client");
......
662 686
                throw new ArgumentNullException("serverHash");
663 687
            Contract.EndContractBlock();
664 688
            
665
            return Task.Factory.Iterate(BlockDownloadIterator(client,account,container, relativeUrl, localPath, serverHash));
689
            return Task.Factory.Iterate(BlockDownloadIterator(accountInfo,client,account,container, relativeUrl, localPath, serverHash));
666 690
        }
667 691

  
668
        private IEnumerable<Task> BlockDownloadIterator(CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath, TreeHash serverHash)
692
        private IEnumerable<Task> BlockDownloadIterator(AccountInfo accountInfo,CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath, TreeHash serverHash)
669 693
        {
670 694
            if (client == null)
671 695
                throw new ArgumentNullException("client");
......
682 706
            if(serverHash==null)
683 707
                throw new ArgumentNullException("serverHash");
684 708
            Contract.EndContractBlock();
685

  
709
            
710
            var fileAgent = GetFileAgent(accountInfo);
686 711
            
687 712
            //Calculate the relative file path for the new file
688 713
            var relativePath = relativeUrl.RelativeUriToFilePath();
689
            var blockUpdater = new BlockUpdater(FileAgent.FragmentsPath, localPath, relativePath, serverHash);
714
            var blockUpdater = new BlockUpdater(fileAgent.FragmentsPath, localPath, relativePath, serverHash);
690 715

  
691 716
            
692 717
                        
b/trunk/Pithos.Core/Pithos.Core.csproj
169 169
  </ItemGroup>
170 170
  <ItemGroup>
171 171
    <Compile Include="Agents\Agent.cs" />
172
    <Compile Include="Agents\AgentLocator.cs" />
172 173
    <Compile Include="Agents\BlockUpdater.cs" />
173 174
    <Compile Include="Agents\CloudTransferAction.cs" />
174 175
    <Compile Include="Agents\FileAgent.cs" />
b/trunk/Pithos.Core/PithosMonitor.cs
48 48
        public WorkflowAgent WorkflowAgent { get; set; }
49 49
        
50 50
        [Import]
51
        public NetworkAgent NetworkAgent { get; set; }
52

  
51
        public NetworkAgent NetworkAgent { get; set; }        
53 52

  
54 53
        public string UserName { get; set; }
55 54
        public string ApiKey { get; set; }
......
79 78
            }
80 79
        }
81 80

  
82
        public string RootPath { get; set; }
81
        private string _rootPath;
82
        public string RootPath
83
        {
84
            get { return _rootPath; }
85
            set
86
            {
87
                _rootPath = value.ToLower();
88
            }
89
        }
83 90

  
84 91

  
85 92
        CancellationTokenSource _cancellationSource;
......
121 128
            StatusKeeper.BlockSize = _blockSize;
122 129
            
123 130
            StatusKeeper.StartProcessing(_cancellationSource.Token);
124
            IndexLocalFiles(RootPath);
125
            StartWatcherAgent(RootPath);
131
            IndexLocalFiles();
132
            StartWatcherAgent();
126 133

  
127 134
            StartNetworkAgent();
128 135

  
......
172 179
            return null;
173 180
        }
174 181

  
175
        private void IndexLocalFiles(string path)
182
        private void IndexLocalFiles()
176 183
        {
177 184
            StatusNotification.NotifyChange("Indexing Local Files",TraceLevel.Info);
178 185
            using (log4net.ThreadContext.Stacks["Monitor"].Push("Indexing local files"))
......
181 188
                try
182 189
                {
183 190
                    var fragmentsPath = Path.Combine(RootPath, FolderConstants.FragmentsFolder);
184
                    var directory = new DirectoryInfo(path);
191
                    var directory = new DirectoryInfo(RootPath);
185 192
                    var files =
186 193
                        from file in directory.EnumerateFiles("*", SearchOption.AllDirectories)
187 194
                        where !file.FullName.StartsWith(fragmentsPath, StringComparison.InvariantCultureIgnoreCase) &&
......
275 282
            NetworkAgent.AddAccount(_accountInfo);
276 283

  
277 284
            NetworkAgent.StatusNotification = StatusNotification;
278
            
279

  
285
                        
280 286
            NetworkAgent.Start();
281 287

  
282 288
            NetworkAgent.ProcessRemoteFiles();
......
307 313
       
308 314

  
309 315

  
310
        private void StartWatcherAgent(string path)
316
        private void StartWatcherAgent()
311 317
        {
318
            AgentLocator<FileAgent>.Register(FileAgent,RootPath);
319

  
312 320
            FileAgent.StatusKeeper = StatusKeeper;
313 321
            FileAgent.Workflow = Workflow;
314 322
            FileAgent.FragmentsPath = Path.Combine(RootPath, FolderConstants.FragmentsFolder);
315
            FileAgent.Start(_accountInfo,path);
323
            FileAgent.Start(_accountInfo, RootPath);
316 324
        }
317 325

  
318 326
        public void Stop()
319
        {          
327
        {
328
            AgentLocator<FileAgent>.Remove(RootPath);
329

  
320 330
            if (FileAgent!=null)
321 331
                FileAgent.Stop();
322 332
            FileAgent = null;
b/trunk/Pithos.Network/RestClient.cs
275 275
                {
276 276
                    Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc);
277 277
                }
278
                throw;
278
                throw exc;
279 279

  
280 280
            }
281 281
            catch(Exception ex)

Also available in: Unified diff