Revision c28a075a
b/trunk/Pithos.Client.WPF/PithosAccount.cs | ||
---|---|---|
52 | 52 |
Log.InfoFormat("[RETRIEVE] Listening at {0}", listenerUrl); |
53 | 53 |
|
54 | 54 |
listener.Start(); |
55 |
|
|
56 |
var startListening = Task.Factory.FromAsync<HttpListenerContext>(listener.BeginGetContext, listener.EndGetContext, null) |
|
57 |
.WithTimeout(TimeSpan.FromMinutes(5)); |
|
55 | 58 |
|
56 |
var task = Task.Factory.FromAsync<HttpListenerContext>(listener.BeginGetContext, listener.EndGetContext, null) |
|
57 |
.WithTimeout(TimeSpan.FromMinutes(1)) |
|
58 |
.ContinueWith(tc => |
|
59 |
var receiveCredentials=startListening.ContinueWith(tc => |
|
59 | 60 |
{ |
60 | 61 |
try |
61 | 62 |
{ |
... | ... | |
96 | 97 |
Log.InfoFormat("[RETRIEVE] Open Browser at {0}", retrieveUri); |
97 | 98 |
Process.Start(retrieveUri.ToString()); |
98 | 99 |
|
99 |
return task;
|
|
100 |
return receiveCredentials;
|
|
100 | 101 |
} |
101 | 102 |
|
102 | 103 |
private static void Respond(HttpListenerContext context) |
b/trunk/Pithos.Client.WPF/ShellViewModel.cs | ||
---|---|---|
1 | 1 |
using System.Collections.Concurrent; |
2 | 2 |
using System.ComponentModel.Composition; |
3 | 3 |
using System.Diagnostics; |
4 |
using System.Diagnostics.Contracts; |
|
4 | 5 |
using System.IO; |
6 |
using System.Net; |
|
5 | 7 |
using System.Runtime.InteropServices; |
6 | 8 |
using System.ServiceModel; |
7 | 9 |
using System.ServiceModel.Description; |
... | ... | |
52 | 54 |
_statusChecker = statusChecker; |
53 | 55 |
_events = events; |
54 | 56 |
Settings = settings; |
55 |
|
|
56 |
|
|
57 |
|
|
57 |
|
|
58 | 58 |
UsageMessage = "Using 15% of 50 GB"; |
59 | 59 |
StatusMessage = "In Synch"; |
60 | 60 |
|
... | ... | |
77 | 77 |
if (_monitors.TryGetValue(accountName,out monitor)) |
78 | 78 |
{ |
79 | 79 |
//If the account is active |
80 |
if (account.IsActive) |
|
80 |
if (account.IsActive)
|
|
81 | 81 |
//Start the monitor. It's OK to start an already started monitor, |
82 | 82 |
//it will just ignore the call |
83 | 83 |
monitor.Start(); |
... | ... | |
268 | 268 |
|
269 | 269 |
|
270 | 270 |
|
271 |
private Task StartMonitor(PithosMonitor monitor) |
|
271 |
private Task StartMonitor(PithosMonitor monitor,int retries=0)
|
|
272 | 272 |
{ |
273 | 273 |
return Task.Factory.StartNew(() => |
274 | 274 |
{ |
... | ... | |
277 | 277 |
try |
278 | 278 |
{ |
279 | 279 |
Log.InfoFormat("Start Monitoring {0}", monitor.UserName); |
280 |
|
|
280 | 281 |
monitor.Start(); |
281 | 282 |
} |
283 |
catch (WebException exc) |
|
284 |
{ |
|
285 |
if (AbandonRetry(monitor, retries)) |
|
286 |
return; |
|
287 |
|
|
288 |
if (IsUnauthorized(exc)) |
|
289 |
{ |
|
290 |
var message = String.Format("API Key Expired for {0}. Starting Renewal",monitor.UserName); |
|
291 |
Log.Error(message,exc); |
|
292 |
TryAuthorize(monitor,retries); |
|
293 |
} |
|
294 |
else |
|
295 |
{ |
|
296 |
TryLater(monitor, exc,retries); |
|
297 |
} |
|
298 |
} |
|
282 | 299 |
catch (Exception exc) |
283 | 300 |
{ |
284 |
var message = |
|
285 |
String.Format("An exception occured. Can't start monitoring\nWill retry in 10 seconds"); |
|
286 |
Task.Factory.StartNewDelayed(10000, () => StartMonitor(monitor)); |
|
287 |
_events.Publish(new Notification {Title = "Error", Message = message, Level = TraceLevel.Error}); |
|
288 |
Log.Error(message, exc); |
|
301 |
if (AbandonRetry(monitor, retries)) |
|
302 |
return; |
|
303 |
|
|
304 |
TryLater(monitor,exc,retries); |
|
289 | 305 |
} |
290 | 306 |
} |
291 | 307 |
}); |
292 | 308 |
} |
293 | 309 |
|
310 |
private bool AbandonRetry(PithosMonitor monitor, int retries) |
|
311 |
{ |
|
312 |
if (retries > 1) |
|
313 |
{ |
|
314 |
var message = String.Format("Monitoring of account {0} has failed too many times. Will not retry", |
|
315 |
monitor.UserName); |
|
316 |
_events.Publish(new Notification |
|
317 |
{Title = "Account monitoring failed", Message = message, Level = TraceLevel.Error}); |
|
318 |
return true; |
|
319 |
} |
|
320 |
return false; |
|
321 |
} |
|
322 |
|
|
323 |
|
|
324 |
private Task TryAuthorize(PithosMonitor monitor,int retries) |
|
325 |
{ |
|
326 |
_events.Publish(new Notification { Title = "Authorization failed", Message = "Your API Key has probably expired. You will be directed to a page where you can renew it", Level = TraceLevel.Error }); |
|
327 |
|
|
328 |
var authorize= PithosAccount.RetrieveCredentialsAsync(Settings.PithosSite); |
|
329 |
|
|
330 |
return authorize.ContinueWith(t => |
|
331 |
{ |
|
332 |
if (t.IsFaulted) |
|
333 |
{ |
|
334 |
string message = String.Format("API Key retrieval for {0} failed", monitor.UserName); |
|
335 |
Log.Error(message,t.Exception.InnerException); |
|
336 |
_events.Publish(new Notification { Title = "Authorization failed", Message = message, Level = TraceLevel.Error }); |
|
337 |
return; |
|
338 |
} |
|
339 |
var credentials = t.Result; |
|
340 |
var account =Settings.Accounts.FirstOrDefault(act => act.AccountName == credentials.UserName); |
|
341 |
account.ApiKey = credentials.Password; |
|
342 |
monitor.ApiKey = credentials.Password; |
|
343 |
Task.Factory.StartNewDelayed(10000, () => StartMonitor(monitor,retries+1)); |
|
344 |
}); |
|
345 |
} |
|
346 |
|
|
347 |
private static bool IsUnauthorized(WebException exc) |
|
348 |
{ |
|
349 |
if (exc==null) |
|
350 |
throw new ArgumentNullException("exc"); |
|
351 |
Contract.EndContractBlock(); |
|
352 |
|
|
353 |
var response = exc.Response as HttpWebResponse; |
|
354 |
if (response == null) |
|
355 |
return false; |
|
356 |
return (response.StatusCode == HttpStatusCode.Unauthorized); |
|
357 |
} |
|
358 |
|
|
359 |
private void TryLater(PithosMonitor monitor, Exception exc,int retries) |
|
360 |
{ |
|
361 |
var message = String.Format("An exception occured. Can't start monitoring\nWill retry in 10 seconds"); |
|
362 |
Task.Factory.StartNewDelayed(10000, () => StartMonitor(monitor,retries+1)); |
|
363 |
_events.Publish(new Notification |
|
364 |
{Title = "Error", Message = message, Level = TraceLevel.Error}); |
|
365 |
Log.Error(message, exc); |
|
366 |
} |
|
367 |
|
|
294 | 368 |
|
295 | 369 |
public void NotifyChange(string status, TraceLevel level=TraceLevel.Info) |
296 | 370 |
{ |
b/trunk/Pithos.Core.Test/NetworkAgentTest.cs | ||
---|---|---|
87 | 87 |
File.Delete(filePath); |
88 | 88 |
|
89 | 89 |
var newHash = client.GetHashMap(null, FolderConstants.PithosContainer, fileName).Result; |
90 |
agent.DownloadWithBlocks(client, account, FolderConstants.PithosContainer, new Uri(fileName, UriKind.Relative), filePath, newHash) |
|
90 |
agent.DownloadWithBlocks(accountInfo, client, account, FolderConstants.PithosContainer, new Uri(fileName, UriKind.Relative), filePath, newHash)
|
|
91 | 91 |
.Wait(); |
92 | 92 |
|
93 | 93 |
Assert.IsTrue(File.Exists(filePath)); |
b/trunk/Pithos.Core/Agents/AgentLocator.cs | ||
---|---|---|
1 |
using System; |
|
2 |
using System.Collections.Concurrent; |
|
3 |
using System.Collections.Generic; |
|
4 |
using System.Linq; |
|
5 |
using System.Text; |
|
6 |
|
|
7 |
namespace Pithos.Core.Agents |
|
8 |
{ |
|
9 |
static class AgentLocator<T> where T:class |
|
10 |
{ |
|
11 |
static ConcurrentDictionary<string, WeakReference> _agents = new ConcurrentDictionary<string, WeakReference>(); |
|
12 |
public static void Register(T agent,string key) |
|
13 |
{ |
|
14 |
_agents[key] = new WeakReference(agent); |
|
15 |
} |
|
16 |
|
|
17 |
public static T Get(string key) |
|
18 |
{ |
|
19 |
return _agents[key].Target as T; |
|
20 |
} |
|
21 |
|
|
22 |
public static bool TryGet(string key, out T value) |
|
23 |
{ |
|
24 |
WeakReference target; |
|
25 |
var exists = _agents.TryGetValue(key, out target); |
|
26 |
value = target.Target as T; |
|
27 |
return exists; |
|
28 |
} |
|
29 |
|
|
30 |
public static void Remove(string key) |
|
31 |
{ |
|
32 |
WeakReference target; |
|
33 |
_agents.TryRemove(key, out target); |
|
34 |
} |
|
35 |
} |
|
36 |
} |
b/trunk/Pithos.Core/Agents/FileAgent.cs | ||
---|---|---|
14 | 14 |
|
15 | 15 |
namespace Pithos.Core.Agents |
16 | 16 |
{ |
17 |
[Export,PartCreationPolicy(CreationPolicy.NonShared)]
|
|
17 |
[Export] |
|
18 | 18 |
public class FileAgent |
19 | 19 |
{ |
20 | 20 |
Agent<WorkflowState> _agent; |
b/trunk/Pithos.Core/Agents/NetworkAgent.cs | ||
---|---|---|
24 | 24 |
public IStatusKeeper StatusKeeper { get; set; } |
25 | 25 |
|
26 | 26 |
public IStatusNotification StatusNotification { get; set; } |
27 |
/* |
|
27 | 28 |
[Import] |
28 | 29 |
public FileAgent FileAgent {get;set;} |
30 |
*/ |
|
29 | 31 |
|
30 | 32 |
/* public int BlockSize { get; set; } |
31 | 33 |
public string BlockHash { get; set; }*/ |
... | ... | |
320 | 322 |
select ProcessAccountFiles(accountInfo, since); |
321 | 323 |
var process=Task.Factory.Iterate(tasks); |
322 | 324 |
|
323 |
return process.ContinueWith(t=> |
|
324 |
ProcessRemoteFiles(nextSince)); |
|
325 |
return process.ContinueWith(t => |
|
326 |
{ |
|
327 |
if (t.IsFaulted) |
|
328 |
{ |
|
329 |
Log.Error("Error while processing accounts"); |
|
330 |
t.Exception.Handle(exc=> |
|
331 |
{ |
|
332 |
Log.Error("Details:", exc); |
|
333 |
return true; |
|
334 |
}); |
|
335 |
} |
|
336 |
ProcessRemoteFiles(nextSince); |
|
337 |
}); |
|
325 | 338 |
} |
326 | 339 |
}); |
327 | 340 |
} |
... | ... | |
429 | 442 |
if (remote==null) |
430 | 443 |
throw new ArgumentNullException(); |
431 | 444 |
Contract.EndContractBlock(); |
445 |
var fileAgent = GetFileAgent(accountInfo); |
|
432 | 446 |
|
433 | 447 |
//In order to avoid multiple iterations over the files, we iterate only once |
434 | 448 |
//over the remote files |
... | ... | |
436 | 450 |
{ |
437 | 451 |
var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName); |
438 | 452 |
//and remove any matching objects from the list, adding them to the commonObjects list |
439 |
if (FileAgent.Exists(relativePath)) |
|
453 |
|
|
454 |
if (fileAgent.Exists(relativePath)) |
|
440 | 455 |
{ |
441 |
var localFile = FileAgent.GetFileInfo(relativePath);
|
|
456 |
var localFile = fileAgent.GetFileInfo(relativePath);
|
|
442 | 457 |
var state = FileState.FindByFilePath(localFile.FullName); |
443 | 458 |
//Common files should be checked on a per-case basis to detect differences, which is newer |
444 | 459 |
|
... | ... | |
461 | 476 |
} |
462 | 477 |
} |
463 | 478 |
|
479 |
private static FileAgent GetFileAgent(AccountInfo accountInfo) |
|
480 |
{ |
|
481 |
return AgentLocator<FileAgent>.Get(accountInfo.AccountPath); |
|
482 |
} |
|
483 |
|
|
464 | 484 |
private void ProcessDeletedFiles(AccountInfo accountInfo,IEnumerable<ObjectInfo> trashObjects) |
465 | 485 |
{ |
486 |
var fileAgent = GetFileAgent(accountInfo); |
|
466 | 487 |
foreach (var trashObject in trashObjects) |
467 | 488 |
{ |
468 | 489 |
var relativePath = trashObject.RelativeUrlToFilePath(accountInfo.UserName); |
469 | 490 |
//and remove any matching objects from the list, adding them to the commonObjects list |
470 |
FileAgent.Delete(relativePath);
|
|
491 |
fileAgent.Delete(relativePath);
|
|
471 | 492 |
} |
472 | 493 |
} |
473 | 494 |
|
... | ... | |
514 | 535 |
if (Path.IsPathRooted(fileName)) |
515 | 536 |
throw new ArgumentException("The fileName should not be rooted","fileName"); |
516 | 537 |
Contract.EndContractBlock(); |
538 |
|
|
539 |
var fileAgent = GetFileAgent(accountInfo); |
|
517 | 540 |
|
518 | 541 |
using ( log4net.ThreadContext.Stacks["DeleteCloudFile"].Push("Delete")) |
519 | 542 |
{ |
520 |
var info = FileAgent.GetFileInfo(fileName);
|
|
543 |
var info = fileAgent.GetFileInfo(fileName);
|
|
521 | 544 |
var fullPath = info.FullName.ToLower(); |
522 | 545 |
this.StatusKeeper.SetFileOverlayStatus(fullPath, FileOverlayStatus.Modified); |
523 | 546 |
|
... | ... | |
586 | 609 |
//If it's a small file |
587 | 610 |
var downloadTask=(serverHash.Hashes.Count == 1 ) |
588 | 611 |
//Download it in one go |
589 |
? DownloadEntireFile(client, account, container, relativeUrl, localPath) |
|
612 |
? DownloadEntireFile(accountInfo,client, account, container, relativeUrl, localPath)
|
|
590 | 613 |
//Otherwise download it block by block |
591 |
: DownloadWithBlocks(client, account, container, relativeUrl, localPath, serverHash); |
|
614 |
: DownloadWithBlocks(accountInfo,client, account, container, relativeUrl, localPath, serverHash);
|
|
592 | 615 |
|
593 | 616 |
yield return downloadTask; |
594 | 617 |
|
... | ... | |
605 | 628 |
} |
606 | 629 |
|
607 | 630 |
//Download a small file with a single GET operation |
608 |
private Task DownloadEntireFile(CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath) |
|
631 |
private Task DownloadEntireFile(AccountInfo accountInfo, CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath)
|
|
609 | 632 |
{ |
610 | 633 |
if (client == null) |
611 | 634 |
throw new ArgumentNullException("client"); |
... | ... | |
621 | 644 |
throw new ArgumentException("The localPath must be rooted", "localPath"); |
622 | 645 |
Contract.EndContractBlock(); |
623 | 646 |
|
647 |
var fileAgent = GetFileAgent(accountInfo); |
|
624 | 648 |
//Calculate the relative file path for the new file |
625 | 649 |
var relativePath = relativeUrl.RelativeUriToFilePath(); |
626 | 650 |
//The file will be stored in a temporary location while downloading with an extension .download |
627 |
var tempPath = Path.Combine(FileAgent.FragmentsPath, relativePath + ".download");
|
|
651 |
var tempPath = Path.Combine(fileAgent.FragmentsPath, relativePath + ".download");
|
|
628 | 652 |
//Make sure the target folder exists. DownloadFileTask will not create the folder |
629 | 653 |
var directoryPath = Path.GetDirectoryName(tempPath); |
630 | 654 |
if (!Directory.Exists(directoryPath)) |
... | ... | |
644 | 668 |
} |
645 | 669 |
|
646 | 670 |
//Download a file asynchronously using blocks |
647 |
public Task DownloadWithBlocks(CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath, TreeHash serverHash) |
|
671 |
public Task DownloadWithBlocks(AccountInfo accountInfo, CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath, TreeHash serverHash)
|
|
648 | 672 |
{ |
649 | 673 |
if (client == null) |
650 | 674 |
throw new ArgumentNullException("client"); |
... | ... | |
662 | 686 |
throw new ArgumentNullException("serverHash"); |
663 | 687 |
Contract.EndContractBlock(); |
664 | 688 |
|
665 |
return Task.Factory.Iterate(BlockDownloadIterator(client,account,container, relativeUrl, localPath, serverHash)); |
|
689 |
return Task.Factory.Iterate(BlockDownloadIterator(accountInfo,client,account,container, relativeUrl, localPath, serverHash));
|
|
666 | 690 |
} |
667 | 691 |
|
668 |
private IEnumerable<Task> BlockDownloadIterator(CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath, TreeHash serverHash) |
|
692 |
private IEnumerable<Task> BlockDownloadIterator(AccountInfo accountInfo,CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath, TreeHash serverHash)
|
|
669 | 693 |
{ |
670 | 694 |
if (client == null) |
671 | 695 |
throw new ArgumentNullException("client"); |
... | ... | |
682 | 706 |
if(serverHash==null) |
683 | 707 |
throw new ArgumentNullException("serverHash"); |
684 | 708 |
Contract.EndContractBlock(); |
685 |
|
|
709 |
|
|
710 |
var fileAgent = GetFileAgent(accountInfo); |
|
686 | 711 |
|
687 | 712 |
//Calculate the relative file path for the new file |
688 | 713 |
var relativePath = relativeUrl.RelativeUriToFilePath(); |
689 |
var blockUpdater = new BlockUpdater(FileAgent.FragmentsPath, localPath, relativePath, serverHash);
|
|
714 |
var blockUpdater = new BlockUpdater(fileAgent.FragmentsPath, localPath, relativePath, serverHash);
|
|
690 | 715 |
|
691 | 716 |
|
692 | 717 |
|
b/trunk/Pithos.Core/Pithos.Core.csproj | ||
---|---|---|
169 | 169 |
</ItemGroup> |
170 | 170 |
<ItemGroup> |
171 | 171 |
<Compile Include="Agents\Agent.cs" /> |
172 |
<Compile Include="Agents\AgentLocator.cs" /> |
|
172 | 173 |
<Compile Include="Agents\BlockUpdater.cs" /> |
173 | 174 |
<Compile Include="Agents\CloudTransferAction.cs" /> |
174 | 175 |
<Compile Include="Agents\FileAgent.cs" /> |
b/trunk/Pithos.Core/PithosMonitor.cs | ||
---|---|---|
48 | 48 |
public WorkflowAgent WorkflowAgent { get; set; } |
49 | 49 |
|
50 | 50 |
[Import] |
51 |
public NetworkAgent NetworkAgent { get; set; } |
|
52 |
|
|
51 |
public NetworkAgent NetworkAgent { get; set; } |
|
53 | 52 |
|
54 | 53 |
public string UserName { get; set; } |
55 | 54 |
public string ApiKey { get; set; } |
... | ... | |
79 | 78 |
} |
80 | 79 |
} |
81 | 80 |
|
82 |
public string RootPath { get; set; } |
|
81 |
private string _rootPath; |
|
82 |
public string RootPath |
|
83 |
{ |
|
84 |
get { return _rootPath; } |
|
85 |
set |
|
86 |
{ |
|
87 |
_rootPath = value.ToLower(); |
|
88 |
} |
|
89 |
} |
|
83 | 90 |
|
84 | 91 |
|
85 | 92 |
CancellationTokenSource _cancellationSource; |
... | ... | |
121 | 128 |
StatusKeeper.BlockSize = _blockSize; |
122 | 129 |
|
123 | 130 |
StatusKeeper.StartProcessing(_cancellationSource.Token); |
124 |
IndexLocalFiles(RootPath);
|
|
125 |
StartWatcherAgent(RootPath);
|
|
131 |
IndexLocalFiles(); |
|
132 |
StartWatcherAgent(); |
|
126 | 133 |
|
127 | 134 |
StartNetworkAgent(); |
128 | 135 |
|
... | ... | |
172 | 179 |
return null; |
173 | 180 |
} |
174 | 181 |
|
175 |
private void IndexLocalFiles(string path)
|
|
182 |
private void IndexLocalFiles() |
|
176 | 183 |
{ |
177 | 184 |
StatusNotification.NotifyChange("Indexing Local Files",TraceLevel.Info); |
178 | 185 |
using (log4net.ThreadContext.Stacks["Monitor"].Push("Indexing local files")) |
... | ... | |
181 | 188 |
try |
182 | 189 |
{ |
183 | 190 |
var fragmentsPath = Path.Combine(RootPath, FolderConstants.FragmentsFolder); |
184 |
var directory = new DirectoryInfo(path);
|
|
191 |
var directory = new DirectoryInfo(RootPath);
|
|
185 | 192 |
var files = |
186 | 193 |
from file in directory.EnumerateFiles("*", SearchOption.AllDirectories) |
187 | 194 |
where !file.FullName.StartsWith(fragmentsPath, StringComparison.InvariantCultureIgnoreCase) && |
... | ... | |
275 | 282 |
NetworkAgent.AddAccount(_accountInfo); |
276 | 283 |
|
277 | 284 |
NetworkAgent.StatusNotification = StatusNotification; |
278 |
|
|
279 |
|
|
285 |
|
|
280 | 286 |
NetworkAgent.Start(); |
281 | 287 |
|
282 | 288 |
NetworkAgent.ProcessRemoteFiles(); |
... | ... | |
307 | 313 |
|
308 | 314 |
|
309 | 315 |
|
310 |
private void StartWatcherAgent(string path)
|
|
316 |
private void StartWatcherAgent() |
|
311 | 317 |
{ |
318 |
AgentLocator<FileAgent>.Register(FileAgent,RootPath); |
|
319 |
|
|
312 | 320 |
FileAgent.StatusKeeper = StatusKeeper; |
313 | 321 |
FileAgent.Workflow = Workflow; |
314 | 322 |
FileAgent.FragmentsPath = Path.Combine(RootPath, FolderConstants.FragmentsFolder); |
315 |
FileAgent.Start(_accountInfo,path);
|
|
323 |
FileAgent.Start(_accountInfo, RootPath);
|
|
316 | 324 |
} |
317 | 325 |
|
318 | 326 |
public void Stop() |
319 |
{ |
|
327 |
{ |
|
328 |
AgentLocator<FileAgent>.Remove(RootPath); |
|
329 |
|
|
320 | 330 |
if (FileAgent!=null) |
321 | 331 |
FileAgent.Stop(); |
322 | 332 |
FileAgent = null; |
b/trunk/Pithos.Network/RestClient.cs | ||
---|---|---|
275 | 275 |
{ |
276 | 276 |
Log.ErrorFormat("[{0}] FAILED for {1} with \n{2}", method, address, exc); |
277 | 277 |
} |
278 |
throw; |
|
278 |
throw exc;
|
|
279 | 279 |
|
280 | 280 |
} |
281 | 281 |
catch(Exception ex) |
Also available in: Unified diff