Revision c53aa229 trunk/Pithos.Core/Agents/NetworkAgent.cs
b/trunk/Pithos.Core/Agents/NetworkAgent.cs | ||
---|---|---|
25 | 25 |
|
26 | 26 |
public IStatusNotification StatusNotification { get; set; } |
27 | 27 |
[Import] |
28 |
public ICloudClient CloudClient { get; set; } |
|
29 |
|
|
30 |
[Import] |
|
31 | 28 |
public FileAgent FileAgent {get;set;} |
32 | 29 |
|
33 |
/* |
|
34 |
[Import] |
|
35 |
public IPithosWorkflow Workflow { get; set; } |
|
36 |
*/ |
|
37 |
|
|
38 |
|
|
39 |
public string PithosContainer { get; set; } |
|
40 |
public string TrashContainer { get; private set; } |
|
41 |
public IList<string> Containers { get; private set; } |
|
42 |
|
|
43 |
public int BlockSize { get; set; } |
|
44 |
public string BlockHash { get; set; } |
|
30 |
/* public int BlockSize { get; set; } |
|
31 |
public string BlockHash { get; set; }*/ |
|
45 | 32 |
|
46 | 33 |
private static readonly ILog Log = LogManager.GetLogger("NetworkAgent"); |
47 | 34 |
|
35 |
private List<AccountInfo> _accounts=new List<AccountInfo>(); |
|
48 | 36 |
|
49 |
public void Start(string pithosContainer, string trashContainer, int blockSize, string blockHash)
|
|
37 |
public void Start(/*int blockSize, string blockHash*/)
|
|
50 | 38 |
{ |
51 |
if (String.IsNullOrWhiteSpace(pithosContainer)) |
|
52 |
throw new ArgumentNullException("pithosContainer"); |
|
53 |
if (String.IsNullOrWhiteSpace(trashContainer)) |
|
54 |
throw new ArgumentNullException("trashContainer"); |
|
39 |
/* |
|
40 |
if (blockSize<0) |
|
41 |
throw new ArgumentOutOfRangeException("blockSize"); |
|
42 |
if (String.IsNullOrWhiteSpace(blockHash)) |
|
43 |
throw new ArgumentOutOfRangeException("blockHash"); |
|
55 | 44 |
Contract.EndContractBlock(); |
45 |
*/ |
|
56 | 46 |
|
57 |
PithosContainer = pithosContainer; |
|
58 |
TrashContainer = trashContainer; |
|
47 |
/* |
|
59 | 48 |
BlockSize = blockSize; |
60 | 49 |
BlockHash = blockHash; |
50 |
*/ |
|
61 | 51 |
|
62 | 52 |
|
63 | 53 |
_agent = Agent<CloudAction>.Start(inbox => |
... | ... | |
77 | 67 |
{ |
78 | 68 |
if (action == null) |
79 | 69 |
throw new ArgumentNullException("action"); |
70 |
if (action.AccountInfo==null) |
|
71 |
throw new ArgumentException("The action.AccountInfo is empty","action"); |
|
80 | 72 |
Contract.EndContractBlock(); |
81 | 73 |
|
74 |
var accountInfo = action.AccountInfo; |
|
75 |
|
|
82 | 76 |
using (log4net.ThreadContext.Stacks["NETWORK"].Push("PROCESS")) |
83 | 77 |
{ |
84 | 78 |
Log.InfoFormat("[ACTION] Start Processing {0}:{1}->{2}", action.Action, action.LocalFile, |
85 | 79 |
action.CloudFile.Name); |
86 | 80 |
|
87 | 81 |
var localFile = action.LocalFile; |
88 |
var cloudFile = action.CloudFile; |
|
82 |
var cloudFile = action.CloudFile;
|
|
89 | 83 |
var downloadPath = (cloudFile == null) |
90 | 84 |
? String.Empty |
91 |
: Path.Combine(FileAgent.RootPath, cloudFile.RelativeUrlToFilePath(CloudClient.UserName));
|
|
85 |
: Path.Combine(accountInfo.AccountPath, cloudFile.RelativeUrlToFilePath(accountInfo.UserName));
|
|
92 | 86 |
|
93 | 87 |
try |
94 | 88 |
{ |
95 |
var account = action.CloudFile.Account ?? CloudClient.UserName;
|
|
96 |
var container = action.CloudFile.Container ?? PithosContainer; |
|
89 |
var account = action.CloudFile.Account ?? accountInfo.UserName;
|
|
90 |
var container = action.CloudFile.Container ?? FolderConstants.PithosContainer;
|
|
97 | 91 |
|
98 | 92 |
switch (action.Action) |
99 | 93 |
{ |
100 | 94 |
case CloudActionType.UploadUnconditional: |
101 |
UploadCloudFile(account, container, localFile, action.LocalHash.Value, action.TopHash.Value); |
|
95 |
UploadCloudFile(accountInfo,account, container, localFile, action.LocalHash.Value, action.TopHash.Value);
|
|
102 | 96 |
break; |
103 | 97 |
case CloudActionType.DownloadUnconditional: |
104 | 98 |
|
105 |
DownloadCloudFile(account, container, new Uri(cloudFile.Name, UriKind.Relative), |
|
99 |
DownloadCloudFile(accountInfo, account, container, new Uri(cloudFile.Name, UriKind.Relative),
|
|
106 | 100 |
downloadPath); |
107 | 101 |
break; |
108 | 102 |
case CloudActionType.DeleteCloud: |
109 |
DeleteCloudFile(account, container, cloudFile.Name); |
|
103 |
DeleteCloudFile(accountInfo, account, container, cloudFile.Name);
|
|
110 | 104 |
break; |
111 | 105 |
case CloudActionType.RenameCloud: |
112 | 106 |
var moveAction = (CloudMoveAction)action; |
113 |
RenameCloudFile(account, container, moveAction.OldFileName, moveAction.NewPath, |
|
107 |
RenameCloudFile(accountInfo, account, container, moveAction.OldFileName, moveAction.NewPath,
|
|
114 | 108 |
moveAction.NewFileName); |
115 | 109 |
break; |
116 | 110 |
case CloudActionType.MustSynch: |
... | ... | |
118 | 112 |
if (!File.Exists(downloadPath)) |
119 | 113 |
{ |
120 | 114 |
var cloudUri = new Uri(action.CloudFile.Name, UriKind.Relative); |
121 |
DownloadCloudFile(account, container, cloudUri, downloadPath); |
|
115 |
DownloadCloudFile(accountInfo, account, container, cloudUri, downloadPath);
|
|
122 | 116 |
} |
123 | 117 |
else |
124 | 118 |
{ |
125 |
SyncFiles(action); |
|
119 |
SyncFiles(accountInfo, action);
|
|
126 | 120 |
} |
127 | 121 |
break; |
128 | 122 |
} |
... | ... | |
137 | 131 |
{ |
138 | 132 |
Log.ErrorFormat("{0} : {1} -> {2} failed because the file was not found.\n Rescheduling a delete", |
139 | 133 |
action.Action, action.LocalFile, action.CloudFile, exc); |
140 |
Post(new CloudDeleteAction(action.CloudFile,action.FileState)); |
|
134 |
Post(new CloudDeleteAction(accountInfo,action.CloudFile,action.FileState));
|
|
141 | 135 |
} |
142 | 136 |
catch (Exception exc) |
143 | 137 |
{ |
... | ... | |
150 | 144 |
} |
151 | 145 |
} |
152 | 146 |
|
153 |
private void SyncFiles(CloudAction action) |
|
147 |
private void SyncFiles(AccountInfo accountInfo,CloudAction action)
|
|
154 | 148 |
{ |
149 |
if (accountInfo == null) |
|
150 |
throw new ArgumentNullException("accountInfo"); |
|
155 | 151 |
if (action==null) |
156 | 152 |
throw new ArgumentNullException("action"); |
157 | 153 |
if (action.LocalFile==null) |
... | ... | |
168 | 164 |
|
169 | 165 |
var account = cloudFile.Account; |
170 | 166 |
//Use "pithos" by default if no container is specified |
171 |
var container = cloudFile.Container ?? PithosContainer; |
|
167 |
var container = cloudFile.Container ?? FolderConstants.PithosContainer;
|
|
172 | 168 |
|
173 | 169 |
var cloudUri = new Uri(cloudFile.Name, UriKind.Relative); |
174 | 170 |
var cloudHash = cloudFile.Hash.ToLower(); |
... | ... | |
192 | 188 |
if (lastUpTime <= lastLocalTime) |
193 | 189 |
{ |
194 | 190 |
//It probably means it was changed while the app was down |
195 |
UploadCloudFile(account, container, localFile, action.LocalHash.Value, |
|
191 |
UploadCloudFile(accountInfo,account, container, localFile, action.LocalHash.Value,
|
|
196 | 192 |
action.TopHash.Value); |
197 | 193 |
} |
198 | 194 |
else |
... | ... | |
204 | 200 |
{ |
205 | 201 |
case FileStatus.Unchanged: |
206 | 202 |
//If the local file's status is Unchanged, we can go on and download the newer cloud file |
207 |
DownloadCloudFile(account, container,cloudUri,downloadPath); |
|
203 |
DownloadCloudFile(accountInfo,account, container,cloudUri,downloadPath);
|
|
208 | 204 |
break; |
209 | 205 |
case FileStatus.Modified: |
210 | 206 |
//If the local file is Modified, we may have a conflict. In this case we should mark the file as Conflict |
... | ... | |
273 | 269 |
{ |
274 | 270 |
if (cloudAction == null) |
275 | 271 |
throw new ArgumentNullException("cloudAction"); |
272 |
if (cloudAction.AccountInfo==null) |
|
273 |
throw new ArgumentException("The CloudAction.AccountInfo is empty","cloudAction"); |
|
276 | 274 |
Contract.EndContractBlock(); |
277 | 275 |
|
278 | 276 |
//If the action targets a local file, add a treehash calculation |
279 | 277 |
if (cloudAction.LocalFile != null) |
280 | 278 |
{ |
281 |
|
|
282 |
if (cloudAction.LocalFile.Length>BlockSize) |
|
283 |
cloudAction.TopHash = new Lazy<string>(() => Signature.CalculateTreeHashAsync(cloudAction.LocalFile,
|
|
284 |
BlockSize, BlockHash).Result
|
|
279 |
var accountInfo = cloudAction.AccountInfo; |
|
280 |
if (cloudAction.LocalFile.Length>accountInfo.BlockSize)
|
|
281 |
cloudAction.TopHash = new Lazy<string>(() => Signature.CalculateTreeHashAsync(cloudAction.LocalFile, |
|
282 |
accountInfo.BlockSize, accountInfo.BlockHash).Result
|
|
285 | 283 |
.TopHash.ToHashString()); |
286 | 284 |
else |
287 | 285 |
{ |
... | ... | |
305 | 303 |
} |
306 | 304 |
} |
307 | 305 |
|
306 |
|
|
307 |
|
|
308 | 308 |
//Remote files are polled periodically. Any changes are processed |
309 |
public Task ProcessRemoteFiles(string accountPath,DateTime? since=null) |
|
309 |
public Task ProcessRemoteFiles(DateTime? since=null) |
|
310 |
{ |
|
311 |
return Task<Task>.Factory.StartNewDelayed(10000, () => |
|
312 |
{ |
|
313 |
using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push("All accounts")) |
|
314 |
{ |
|
315 |
//Next time we will check for all changes since the current check minus 1 second |
|
316 |
//This is done to ensure there are no discrepancies due to clock differences |
|
317 |
DateTime nextSince = DateTime.Now.AddSeconds(-1); |
|
318 |
|
|
319 |
var tasks=from accountInfo in _accounts |
|
320 |
select ProcessAccountFiles(accountInfo, since); |
|
321 |
var process=Task.Factory.Iterate(tasks); |
|
322 |
|
|
323 |
return process.ContinueWith(t=> |
|
324 |
ProcessRemoteFiles(nextSince)); |
|
325 |
} |
|
326 |
}); |
|
327 |
} |
|
328 |
|
|
329 |
public Task ProcessAccountFiles(AccountInfo accountInfo,DateTime? since=null) |
|
310 | 330 |
{ |
311 |
if (String.IsNullOrWhiteSpace(accountPath)) |
|
312 |
throw new ArgumentNullException(accountPath); |
|
331 |
if (accountInfo==null) |
|
332 |
throw new ArgumentNullException("accountInfo"); |
|
333 |
if (String.IsNullOrWhiteSpace(accountInfo.AccountPath)) |
|
334 |
throw new ArgumentException("The AccountInfo.AccountPath is empty","accountInfo"); |
|
313 | 335 |
Contract.EndContractBlock(); |
314 | 336 |
|
315 |
using (log4net.ThreadContext.Stacks["SCHEDULE"].Push("Retrieve Remote"))
|
|
337 |
using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))
|
|
316 | 338 |
{ |
317 |
Log.Info("[LISTENER] Scheduled"); |
|
339 |
Log.Info("Scheduled"); |
|
340 |
var client=new CloudFilesClient(accountInfo); |
|
318 | 341 |
|
319 | 342 |
//Get the list of server objects changed since the last check |
320 |
var listObjects = Task<IList<ObjectInfo>>.Factory.StartNewDelayed(10000, () =>
|
|
321 |
CloudClient.ListObjects(CloudClient.UserName, PithosContainer, since));
|
|
343 |
var listObjects = Task<IList<ObjectInfo>>.Factory.StartNew(() =>
|
|
344 |
client.ListObjects(accountInfo.UserName, FolderConstants.PithosContainer, since));
|
|
322 | 345 |
//Get the list of deleted objects since the last check |
323 |
var listTrash = Task<IList<ObjectInfo>>.Factory.StartNewDelayed(10000, () =>
|
|
324 |
CloudClient.ListObjects(CloudClient.UserName, TrashContainer, since));
|
|
346 |
var listTrash = Task<IList<ObjectInfo>>.Factory.StartNew(() =>
|
|
347 |
client.ListObjects(accountInfo.UserName, FolderConstants.TrashContainer, since));
|
|
325 | 348 |
|
326 |
var listShared = Task<IList<ObjectInfo>>.Factory.StartNewDelayed(10000, () =>
|
|
327 |
CloudClient.ListSharedObjects(since));
|
|
349 |
var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(() =>
|
|
350 |
client.ListSharedObjects(since));
|
|
328 | 351 |
|
329 | 352 |
var listAll = Task.Factory.TrackedSequence( |
330 | 353 |
() => listObjects, |
331 | 354 |
() => listTrash, |
332 | 355 |
() => listShared); |
333 | 356 |
|
334 |
//Next time we will check for all changes since the current check minus 1 second |
|
335 |
//This is done to ensure there are no discrepancies due to clock differences |
|
336 |
DateTime nextSince = DateTime.Now.AddSeconds(-1); |
|
337 | 357 |
|
338 | 358 |
|
339 | 359 |
var enqueueFiles = listAll.ContinueWith(task => |
... | ... | |
341 | 361 |
if (task.IsFaulted) |
342 | 362 |
{ |
343 | 363 |
//ListObjects failed at this point, need to reschedule |
344 |
Log.ErrorFormat("[FAIL] ListObjects in ProcessRemoteFiles with {0}", task.Exception); |
|
345 |
ProcessRemoteFiles(accountPath, since); |
|
364 |
Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {0}", accountInfo.UserName,task.Exception); |
|
346 | 365 |
return; |
347 | 366 |
} |
348 | 367 |
using (log4net.ThreadContext.Stacks["SCHEDULE"].Push("Process Results")) |
... | ... | |
356 | 375 |
var realTrash = from trash in trashObjects |
357 | 376 |
where !remoteObjects.Any(info => info.Hash == trash.Hash) |
358 | 377 |
select trash; |
359 |
ProcessDeletedFiles(realTrash); |
|
378 |
ProcessDeletedFiles(accountInfo,realTrash);
|
|
360 | 379 |
|
361 | 380 |
|
362 | 381 |
var remote = from info in remoteObjects.Union(sharedObjects) |
... | ... | |
366 | 385 |
select info; |
367 | 386 |
|
368 | 387 |
//Create a list of actions from the remote files |
369 |
var allActions = ObjectsToActions(remote); |
|
388 |
var allActions = ObjectsToActions(accountInfo,remote);
|
|
370 | 389 |
|
371 | 390 |
//And remove those that are already being processed by the agent |
372 | 391 |
var distinctActions = allActions |
... | ... | |
389 | 408 |
} |
390 | 409 |
}); |
391 | 410 |
|
392 |
var loop = enqueueFiles.ContinueWith(t =>
|
|
411 |
var log = enqueueFiles.ContinueWith(t =>
|
|
393 | 412 |
{ |
394 | 413 |
if (t.IsFaulted) |
395 | 414 |
{ |
... | ... | |
399 | 418 |
{ |
400 | 419 |
Log.Info("[LISTENER] Finished"); |
401 | 420 |
} |
402 |
ProcessRemoteFiles(accountPath, nextSince); |
|
403 |
|
|
404 | 421 |
}); |
405 |
return loop;
|
|
422 |
return log;
|
|
406 | 423 |
} |
407 | 424 |
} |
408 | 425 |
|
409 | 426 |
//Creates an appropriate action for each server file |
410 |
private IEnumerable<CloudAction> ObjectsToActions(IEnumerable<ObjectInfo> remote) |
|
427 |
private IEnumerable<CloudAction> ObjectsToActions(AccountInfo accountInfo,IEnumerable<ObjectInfo> remote)
|
|
411 | 428 |
{ |
412 | 429 |
if (remote==null) |
413 | 430 |
throw new ArgumentNullException(); |
... | ... | |
417 | 434 |
//over the remote files |
418 | 435 |
foreach (var objectInfo in remote) |
419 | 436 |
{ |
420 |
var relativePath = objectInfo.RelativeUrlToFilePath(CloudClient.UserName);
|
|
437 |
var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);
|
|
421 | 438 |
//and remove any matching objects from the list, adding them to the commonObjects list |
422 | 439 |
if (FileAgent.Exists(relativePath)) |
423 | 440 |
{ |
... | ... | |
425 | 442 |
var state = FileState.FindByFilePath(localFile.FullName); |
426 | 443 |
//Common files should be checked on a per-case basis to detect differences, which is newer |
427 | 444 |
|
428 |
yield return new CloudAction(CloudActionType.MustSynch, |
|
429 |
localFile, objectInfo, state, BlockSize, BlockHash); |
|
445 |
yield return new CloudAction(accountInfo,CloudActionType.MustSynch, |
|
446 |
localFile, objectInfo, state, accountInfo.BlockSize, |
|
447 |
accountInfo.BlockHash); |
|
430 | 448 |
} |
431 | 449 |
else |
432 | 450 |
{ |
433 | 451 |
//If there is no match we add them to the localFiles list |
434 | 452 |
//but only if the file is not marked for deletion |
435 |
var targetFile = Path.Combine(FileAgent.RootPath, relativePath);
|
|
453 |
var targetFile = Path.Combine(accountInfo.AccountPath, relativePath);
|
|
436 | 454 |
var fileStatus = StatusKeeper.GetFileStatus(targetFile); |
437 | 455 |
if (fileStatus != FileStatus.Deleted) |
438 | 456 |
{ |
439 | 457 |
//Remote files should be downloaded |
440 |
yield return new CloudDownloadAction(objectInfo); |
|
458 |
yield return new CloudDownloadAction(accountInfo,objectInfo);
|
|
441 | 459 |
} |
442 | 460 |
} |
443 | 461 |
} |
444 | 462 |
} |
445 | 463 |
|
446 |
private void ProcessDeletedFiles(IEnumerable<ObjectInfo> trashObjects) |
|
464 |
private void ProcessDeletedFiles(AccountInfo accountInfo,IEnumerable<ObjectInfo> trashObjects)
|
|
447 | 465 |
{ |
448 | 466 |
foreach (var trashObject in trashObjects) |
449 | 467 |
{ |
450 |
var relativePath = trashObject.RelativeUrlToFilePath(CloudClient.UserName);
|
|
468 |
var relativePath = trashObject.RelativeUrlToFilePath(accountInfo.UserName);
|
|
451 | 469 |
//and remove any matching objects from the list, adding them to the commonObjects list |
452 | 470 |
FileAgent.Delete(relativePath); |
453 | 471 |
} |
454 | 472 |
} |
455 | 473 |
|
456 | 474 |
|
457 |
private void RenameCloudFile(string account, string container,string oldFileName, string newPath, string newFileName) |
|
475 |
private void RenameCloudFile(AccountInfo accountInfo,string account, string container,string oldFileName, string newPath, string newFileName)
|
|
458 | 476 |
{ |
477 |
if (accountInfo==null) |
|
478 |
throw new ArgumentNullException("accountInfo"); |
|
459 | 479 |
if (String.IsNullOrWhiteSpace(account)) |
460 | 480 |
throw new ArgumentNullException("account"); |
461 | 481 |
if (String.IsNullOrWhiteSpace(container)) |
... | ... | |
470 | 490 |
//The local file is already renamed |
471 | 491 |
this.StatusKeeper.SetFileOverlayStatus(newPath, FileOverlayStatus.Modified); |
472 | 492 |
|
473 |
CloudClient.MoveObject(account, container, oldFileName, container, newFileName); |
|
493 |
var client = new CloudFilesClient(accountInfo); |
|
494 |
client.MoveObject(account, container, oldFileName, container, newFileName); |
|
474 | 495 |
|
475 | 496 |
this.StatusKeeper.SetFileStatus(newPath, FileStatus.Unchanged); |
476 | 497 |
this.StatusKeeper.SetFileOverlayStatus(newPath, FileOverlayStatus.Normal); |
477 | 498 |
NativeMethods.RaiseChangeNotification(newPath); |
478 | 499 |
} |
479 | 500 |
|
480 |
private void DeleteCloudFile(string account,string container, string fileName)
|
|
501 |
private void DeleteCloudFile(AccountInfo accountInfo, string account, string container, string fileName)
|
|
481 | 502 |
{ |
503 |
if (accountInfo == null) |
|
504 |
throw new ArgumentNullException("accountInfo"); |
|
482 | 505 |
if (String.IsNullOrWhiteSpace(account)) |
483 | 506 |
throw new ArgumentNullException("account"); |
484 | 507 |
if (String.IsNullOrWhiteSpace(container)) |
... | ... | |
495 | 518 |
using ( log4net.ThreadContext.Stacks["DeleteCloudFile"].Push("Delete")) |
496 | 519 |
{ |
497 | 520 |
var info = FileAgent.GetFileInfo(fileName); |
498 |
var path = info.FullName.ToLower();
|
|
499 |
this.StatusKeeper.SetFileOverlayStatus(path, FileOverlayStatus.Modified);
|
|
521 |
var fullPath = info.FullName.ToLower();
|
|
522 |
this.StatusKeeper.SetFileOverlayStatus(fullPath, FileOverlayStatus.Modified);
|
|
500 | 523 |
|
501 |
CloudClient.DeleteObject(account, container, fileName, TrashContainer); |
|
524 |
var client = new CloudFilesClient(accountInfo); |
|
525 |
client.DeleteObject(account, container, fileName); |
|
502 | 526 |
|
503 |
this.StatusKeeper.ClearFileStatus(path);
|
|
527 |
this.StatusKeeper.ClearFileStatus(fullPath);
|
|
504 | 528 |
} |
505 | 529 |
} |
506 | 530 |
|
507 | 531 |
//Download a file. |
508 |
private void DownloadCloudFile(string account,string container, Uri relativeUrl, string localPath)
|
|
532 |
private void DownloadCloudFile(AccountInfo accountInfo, string account, string container, Uri relativeUrl, string localPath)
|
|
509 | 533 |
{ |
534 |
if (accountInfo == null) |
|
535 |
throw new ArgumentNullException("accountInfo"); |
|
510 | 536 |
if (String.IsNullOrWhiteSpace(account)) |
511 | 537 |
throw new ArgumentNullException("account"); |
512 | 538 |
if (String.IsNullOrWhiteSpace(container)) |
... | ... | |
519 | 545 |
throw new ArgumentException("The localPath must be rooted", "localPath"); |
520 | 546 |
Contract.EndContractBlock(); |
521 | 547 |
|
522 |
var download=Task.Factory.Iterate(DownloadIterator(account,container, relativeUrl, localPath)); |
|
548 |
var download=Task.Factory.Iterate(DownloadIterator(accountInfo,account,container, relativeUrl, localPath));
|
|
523 | 549 |
download.Wait(); |
524 | 550 |
} |
525 | 551 |
|
526 |
private IEnumerable<Task> DownloadIterator(string account,string container, Uri relativeUrl, string localPath)
|
|
552 |
private IEnumerable<Task> DownloadIterator(AccountInfo accountInfo, string account, string container, Uri relativeUrl, string localPath)
|
|
527 | 553 |
{ |
554 |
if (accountInfo == null) |
|
555 |
throw new ArgumentNullException("accountInfo"); |
|
528 | 556 |
if (String.IsNullOrWhiteSpace(account)) |
529 | 557 |
throw new ArgumentNullException("account"); |
530 | 558 |
if (String.IsNullOrWhiteSpace(container)) |
... | ... | |
549 | 577 |
//The file's hashmap will be stored in the same location with the extension .hashmap |
550 | 578 |
//var hashPath = Path.Combine(FileAgent.FragmentsPath, relativePath + ".hashmap"); |
551 | 579 |
|
580 |
var client = new CloudFilesClient(accountInfo); |
|
552 | 581 |
//Retrieve the hashmap from the server |
553 |
var getHashMap = CloudClient.GetHashMap(account, container, url);
|
|
582 |
var getHashMap = client.GetHashMap(account, container, url);
|
|
554 | 583 |
yield return getHashMap; |
555 | 584 |
|
556 | 585 |
var serverHash=getHashMap.Result; |
557 | 586 |
//If it's a small file |
558 | 587 |
var downloadTask=(serverHash.Hashes.Count == 1 ) |
559 | 588 |
//Download it in one go |
560 |
? DownloadEntireFile(account,container, relativeUrl, localPath)
|
|
589 |
? DownloadEntireFile(client, account, container, relativeUrl, localPath)
|
|
561 | 590 |
//Otherwise download it block by block |
562 |
: DownloadWithBlocks(account,container, relativeUrl, localPath, serverHash);
|
|
591 |
: DownloadWithBlocks(client, account, container, relativeUrl, localPath, serverHash);
|
|
563 | 592 |
|
564 | 593 |
yield return downloadTask; |
565 | 594 |
|
566 | 595 |
|
567 | 596 |
//Retrieve the object's metadata |
568 |
var info=CloudClient.GetObjectInfo(account, container, url);
|
|
597 |
var info=client.GetObjectInfo(account, container, url);
|
|
569 | 598 |
//And store it |
570 | 599 |
StatusKeeper.StoreInfo(localPath, info); |
571 | 600 |
|
... | ... | |
576 | 605 |
} |
577 | 606 |
|
578 | 607 |
//Download a small file with a single GET operation |
579 |
private Task DownloadEntireFile(string account,string container, Uri relativeUrl, string localPath)
|
|
608 |
private Task DownloadEntireFile(CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath)
|
|
580 | 609 |
{ |
610 |
if (client == null) |
|
611 |
throw new ArgumentNullException("client"); |
|
581 | 612 |
if (String.IsNullOrWhiteSpace(account)) |
582 | 613 |
throw new ArgumentNullException("account"); |
583 | 614 |
if (String.IsNullOrWhiteSpace(container)) |
... | ... | |
600 | 631 |
Directory.CreateDirectory(directoryPath); |
601 | 632 |
|
602 | 633 |
//Download the object to the temporary location |
603 |
var getObject = CloudClient.GetObject(account, container, relativeUrl.ToString(), tempPath).ContinueWith(t =>
|
|
634 |
var getObject = client.GetObject(account, container, relativeUrl.ToString(), tempPath).ContinueWith(t =>
|
|
604 | 635 |
{ |
605 | 636 |
t.PropagateExceptions(); |
606 | 637 |
//And move it to its actual location once downloading is finished |
... | ... | |
613 | 644 |
} |
614 | 645 |
|
615 | 646 |
//Download a file asynchronously using blocks |
616 |
public Task DownloadWithBlocks(string account,string container, Uri relativeUrl, string localPath, TreeHash serverHash)
|
|
647 |
public Task DownloadWithBlocks(CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath, TreeHash serverHash)
|
|
617 | 648 |
{ |
649 |
if (client == null) |
|
650 |
throw new ArgumentNullException("client"); |
|
618 | 651 |
if (String.IsNullOrWhiteSpace(account)) |
619 | 652 |
throw new ArgumentNullException("account"); |
620 | 653 |
if (String.IsNullOrWhiteSpace(container)) |
... | ... | |
629 | 662 |
throw new ArgumentNullException("serverHash"); |
630 | 663 |
Contract.EndContractBlock(); |
631 | 664 |
|
632 |
return Task.Factory.Iterate(BlockDownloadIterator(account,container, relativeUrl, localPath, serverHash)); |
|
665 |
return Task.Factory.Iterate(BlockDownloadIterator(client,account,container, relativeUrl, localPath, serverHash));
|
|
633 | 666 |
} |
634 |
|
|
635 |
private IEnumerable<Task> BlockDownloadIterator(string account,string container,Uri relativeUrl, string localPath,TreeHash serverHash)
|
|
667 |
|
|
668 |
private IEnumerable<Task> BlockDownloadIterator(CloudFilesClient client, string account, string container, Uri relativeUrl, string localPath, TreeHash serverHash)
|
|
636 | 669 |
{ |
670 |
if (client == null) |
|
671 |
throw new ArgumentNullException("client"); |
|
637 | 672 |
if (String.IsNullOrWhiteSpace(account)) |
638 | 673 |
throw new ArgumentNullException("account"); |
639 | 674 |
if (String.IsNullOrWhiteSpace(container)) |
... | ... | |
656 | 691 |
|
657 | 692 |
|
658 | 693 |
//Calculate the file's treehash |
659 |
var calcHash = Signature.CalculateTreeHashAsync(localPath, this.BlockSize,BlockHash);
|
|
694 |
var calcHash = Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize,serverHash.BlockHash);
|
|
660 | 695 |
yield return calcHash; |
661 | 696 |
var treeHash = calcHash.Result; |
662 | 697 |
|
... | ... | |
675 | 710 |
continue; |
676 | 711 |
} |
677 | 712 |
Log.InfoFormat("[BLOCK GET] START {0} of {1} for {2}", i, upHashes.Length, localPath); |
678 |
var start = i*BlockSize; |
|
713 |
var start = i*serverHash.BlockSize;
|
|
679 | 714 |
//To download the last block just pass a null for the end of the range |
680 | 715 |
long? end = null; |
681 | 716 |
if (i < upHashes.Length - 1 ) |
682 |
end= ((i + 1)*BlockSize) ; |
|
717 |
end= ((i + 1)*serverHash.BlockSize) ;
|
|
683 | 718 |
|
684 | 719 |
//Download the missing block |
685 |
var getBlock = CloudClient.GetBlock(account, container, relativeUrl, start, end);
|
|
720 |
var getBlock = client.GetBlock(account, container, relativeUrl, start, end);
|
|
686 | 721 |
yield return getBlock; |
687 | 722 |
var block = getBlock.Result; |
688 | 723 |
|
... | ... | |
699 | 734 |
} |
700 | 735 |
|
701 | 736 |
|
702 |
private void UploadCloudFile(string account,string container,FileInfo fileInfo, string hash,string topHash)
|
|
737 |
private void UploadCloudFile(AccountInfo accountInfo, string account, string container, FileInfo fileInfo, string hash, string topHash)
|
|
703 | 738 |
{ |
739 |
if (accountInfo == null) |
|
740 |
throw new ArgumentNullException("accountInfo"); |
|
704 | 741 |
if (String.IsNullOrWhiteSpace(account)) |
705 | 742 |
throw new ArgumentNullException("account"); |
706 | 743 |
if (String.IsNullOrWhiteSpace(container)) |
... | ... | |
713 | 750 |
throw new ArgumentNullException("topHash"); |
714 | 751 |
Contract.EndContractBlock(); |
715 | 752 |
|
716 |
var upload = Task.Factory.Iterate(UploadIterator(account,container,fileInfo, hash.ToLower(), topHash.ToLower())); |
|
753 |
var upload = Task.Factory.Iterate(UploadIterator(accountInfo,account,container,fileInfo, hash.ToLower(), topHash.ToLower()));
|
|
717 | 754 |
upload.Wait(); |
718 | 755 |
} |
719 | 756 |
|
720 |
private IEnumerable<Task> UploadIterator(string account,string container,FileInfo fileInfo, string hash,string topHash)
|
|
757 |
private IEnumerable<Task> UploadIterator(AccountInfo accountInfo, string account, string container, FileInfo fileInfo, string hash, string topHash)
|
|
721 | 758 |
{ |
759 |
if (accountInfo == null) |
|
760 |
throw new ArgumentNullException("accountInfo"); |
|
722 | 761 |
if (String.IsNullOrWhiteSpace(account)) |
723 | 762 |
throw new ArgumentNullException("account"); |
724 | 763 |
if (String.IsNullOrWhiteSpace(container)) |
... | ... | |
734 | 773 |
if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)) |
735 | 774 |
yield break; |
736 | 775 |
|
737 |
var url = fileInfo.AsRelativeUrlTo(FileAgent.RootPath);
|
|
776 |
var url = fileInfo.AsRelativeUrlTo(accountInfo.AccountPath);
|
|
738 | 777 |
|
739 | 778 |
var fullFileName = fileInfo.FullName; |
740 | 779 |
using(var gate=NetworkGate.Acquire(fullFileName,NetworkOperation.Uploading)) |
741 | 780 |
{ |
742 | 781 |
//Abort if the file is already being uploaded or downloaded |
743 | 782 |
if (gate.Failed) |
744 |
yield break; |
|
745 |
|
|
783 |
yield break; |
|
746 | 784 |
|
785 |
var client = new CloudFilesClient(accountInfo); |
|
747 | 786 |
//Even if GetObjectInfo times out, we can proceed with the upload |
748 |
var info = CloudClient.GetObjectInfo(account, container, url);
|
|
787 |
var info = client.GetObjectInfo(account, container, url);
|
|
749 | 788 |
var cloudHash = info.Hash.ToLower(); |
750 | 789 |
|
751 | 790 |
//If the file hashes match, abort the upload |
... | ... | |
762 | 801 |
//And then upload it |
763 | 802 |
|
764 | 803 |
//If the file is larger than the block size, try a hashmap PUT |
765 |
if (fileInfo.Length > BlockSize ) |
|
804 |
if (fileInfo.Length > accountInfo.BlockSize )
|
|
766 | 805 |
{ |
767 | 806 |
//To upload using a hashmap |
768 | 807 |
//First, calculate the tree hash |
769 |
var treeHash = Signature.CalculateTreeHashAsync(fileInfo.FullName, BlockSize, BlockHash); |
|
808 |
var treeHash = Signature.CalculateTreeHashAsync(fileInfo.FullName, accountInfo.BlockSize, |
|
809 |
accountInfo.BlockHash); |
|
770 | 810 |
yield return treeHash; |
771 | 811 |
|
772 |
yield return Task.Factory.Iterate(UploadWithHashMap(account,container,fileInfo,url,treeHash)); |
|
812 |
yield return Task.Factory.Iterate(UploadWithHashMap(accountInfo,account,container,fileInfo,url,treeHash));
|
|
773 | 813 |
|
774 | 814 |
} |
775 | 815 |
else |
776 | 816 |
{ |
777 | 817 |
//Otherwise do a regular PUT |
778 |
yield return CloudClient.PutObject(account, container, url, fullFileName, hash);
|
|
818 |
yield return client.PutObject(account, container, url, fullFileName, hash);
|
|
779 | 819 |
} |
780 | 820 |
//If everything succeeds, change the file and overlay status to normal |
781 | 821 |
this.StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal); |
... | ... | |
785 | 825 |
StatusNotification.NotifyChangedFile(fullFileName); |
786 | 826 |
} |
787 | 827 |
|
788 |
public IEnumerable<Task> UploadWithHashMap(string account,string container,FileInfo fileInfo,string url,Task<TreeHash> treeHash) |
|
828 |
public IEnumerable<Task> UploadWithHashMap(AccountInfo accountInfo,string account,string container,FileInfo fileInfo,string url,Task<TreeHash> treeHash)
|
|
789 | 829 |
{ |
830 |
if (accountInfo == null) |
|
831 |
throw new ArgumentNullException("accountInfo"); |
|
790 | 832 |
if (String.IsNullOrWhiteSpace(account)) |
791 | 833 |
throw new ArgumentNullException("account"); |
792 | 834 |
if (String.IsNullOrWhiteSpace(container)) |
... | ... | |
801 | 843 |
|
802 | 844 |
var fullFileName = fileInfo.FullName; |
803 | 845 |
|
846 |
var client = new CloudFilesClient(accountInfo); |
|
804 | 847 |
//Send the hashmap to the server |
805 |
var hashPut = CloudClient.PutHashMap(account, container, url, treeHash.Result);
|
|
848 |
var hashPut = client.PutHashMap(account, container, url, treeHash.Result);
|
|
806 | 849 |
yield return hashPut; |
807 | 850 |
|
808 | 851 |
var missingHashes = hashPut.Result; |
... | ... | |
810 | 853 |
while (missingHashes.Count > 0) |
811 | 854 |
{ |
812 | 855 |
|
813 |
var buffer = new byte[BlockSize]; |
|
856 |
var buffer = new byte[accountInfo.BlockSize];
|
|
814 | 857 |
foreach (var missingHash in missingHashes) |
815 | 858 |
{ |
816 | 859 |
//Find the proper block |
817 | 860 |
var blockIndex = treeHash.Result.HashDictionary[missingHash]; |
818 |
var offset = blockIndex*BlockSize; |
|
861 |
var offset = blockIndex*accountInfo.BlockSize;
|
|
819 | 862 |
|
820 |
var read = fileInfo.Read(buffer, offset, BlockSize); |
|
863 |
var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);
|
|
821 | 864 |
|
822 | 865 |
//And upload the block |
823 |
var postBlock = CloudClient.PostBlock(account, container, buffer, 0, read);
|
|
866 |
var postBlock = client.PostBlock(account, container, buffer, 0, read);
|
|
824 | 867 |
|
825 | 868 |
//We have to handle possible exceptions in a continuation because |
826 | 869 |
//*yield return* can't appear inside a try block |
... | ... | |
831 | 874 |
} |
832 | 875 |
|
833 | 876 |
//Repeat until there are no more missing hashes |
834 |
hashPut = CloudClient.PutHashMap(account, container, url, treeHash.Result);
|
|
877 |
hashPut = client.PutHashMap(account, container, url, treeHash.Result);
|
|
835 | 878 |
yield return hashPut; |
836 | 879 |
missingHashes = hashPut.Result; |
837 | 880 |
} |
838 | 881 |
} |
839 | 882 |
|
840 | 883 |
|
884 |
public void AddAccount(AccountInfo accountInfo) |
|
885 |
{ |
|
886 |
if (!_accounts.Contains(accountInfo)) |
|
887 |
_accounts.Add(accountInfo); |
|
888 |
} |
|
841 | 889 |
} |
842 | 890 |
|
843 | 891 |
|
Also available in: Unified diff