Revision 4f6d51d4 trunk/Pithos.Core/Agents/NetworkAgent.cs
| b/trunk/Pithos.Core/Agents/NetworkAgent.cs | ||
|---|---|---|
| 1 | 1 |
using System; |
| 2 |
using System.Collections.Concurrent; |
|
| 2 | 3 |
using System.Collections.Generic; |
| 3 | 4 |
using System.ComponentModel.Composition; |
| 4 | 5 |
using System.Diagnostics; |
| ... | ... | |
| 26 | 27 |
|
| 27 | 28 |
private static readonly ILog Log = LogManager.GetLogger("NetworkAgent");
|
| 28 | 29 |
|
| 29 |
private readonly List<AccountInfo> _accounts=new List<AccountInfo>();
|
|
| 30 |
private readonly ConcurrentBag<AccountInfo> _accounts = new ConcurrentBag<AccountInfo>();
|
|
| 30 | 31 |
|
| 31 | 32 |
public void Start() |
| 32 | 33 |
{
|
| ... | ... | |
| 81 | 82 |
RenameCloudFile(accountInfo, moveAction); |
| 82 | 83 |
break; |
| 83 | 84 |
case CloudActionType.MustSynch: |
| 84 |
|
|
| 85 |
if (!File.Exists(downloadPath)) |
|
| 85 |
if (!File.Exists(downloadPath) && !Directory.Exists(downloadPath)) |
|
| 86 | 86 |
{
|
| 87 | 87 |
await DownloadCloudFile(accountInfo, cloudFile, downloadPath); |
| 88 | 88 |
} |
| ... | ... | |
| 221 | 221 |
Contract.EndContractBlock(); |
| 222 | 222 |
|
| 223 | 223 |
//If the action targets a local file, add a treehash calculation |
| 224 |
if (cloudAction.LocalFile != null) |
|
| 224 |
if (cloudAction.LocalFile as FileInfo != null)
|
|
| 225 | 225 |
{
|
| 226 | 226 |
var accountInfo = cloudAction.AccountInfo; |
| 227 |
if (!Directory.Exists(cloudAction.LocalFile.FullName)) |
|
| 227 |
var localFile = (FileInfo) cloudAction.LocalFile; |
|
| 228 |
if (localFile.Length > accountInfo.BlockSize) |
|
| 229 |
cloudAction.TopHash = |
|
| 230 |
new Lazy<string>(() => Signature.CalculateTreeHashAsync(localFile, |
|
| 231 |
accountInfo.BlockSize, |
|
| 232 |
accountInfo.BlockHash).Result |
|
| 233 |
.TopHash.ToHashString()); |
|
| 234 |
else |
|
| 228 | 235 |
{
|
| 229 |
if (cloudAction.LocalFile.Length > accountInfo.BlockSize) |
|
| 230 |
cloudAction.TopHash = |
|
| 231 |
new Lazy<string>(() => Signature.CalculateTreeHashAsync(cloudAction.LocalFile, |
|
| 232 |
accountInfo.BlockSize, |
|
| 233 |
accountInfo.BlockHash).Result |
|
| 234 |
.TopHash.ToHashString()); |
|
| 235 |
else |
|
| 236 |
{
|
|
| 237 |
cloudAction.TopHash = new Lazy<string>(() => cloudAction.LocalHash.Value); |
|
| 238 |
} |
|
| 236 |
cloudAction.TopHash = new Lazy<string>(() => cloudAction.LocalHash.Value); |
|
| 239 | 237 |
} |
| 240 |
|
|
| 238 |
} |
|
| 239 |
else |
|
| 240 |
{
|
|
| 241 |
//The hash for a directory is the empty string |
|
| 242 |
cloudAction.TopHash = new Lazy<string>(() => String.Empty); |
|
| 241 | 243 |
} |
| 242 | 244 |
_agent.Post(cloudAction); |
| 243 | 245 |
} |
| ... | ... | |
| 274 | 276 |
var tasks = from accountInfo in _accounts |
| 275 | 277 |
select ProcessAccountFiles(accountInfo, since); |
| 276 | 278 |
|
| 277 |
await TaskEx.WhenAll(tasks); |
|
| 279 |
await TaskEx.WhenAll(tasks.ToList());
|
|
| 278 | 280 |
|
| 279 | 281 |
ProcessRemoteFiles(nextSince); |
| 280 | 282 |
} |
| ... | ... | |
| 377 | 379 |
} |
| 378 | 380 |
} |
| 379 | 381 |
|
| 380 |
private static void CreateContainerFolders(AccountInfo accountInfo, IList<ContainerInfo> containers)
|
|
| 382 |
private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers)
|
|
| 381 | 383 |
{
|
| 382 | 384 |
var containerPaths = from container in containers |
| 383 | 385 |
let containerPath = Path.Combine(accountInfo.AccountPath, container.Name) |
| ... | ... | |
| 407 | 409 |
|
| 408 | 410 |
if (fileAgent.Exists(relativePath)) |
| 409 | 411 |
{
|
| 412 |
//If a directory object already exists, we don't need to perform any other action |
|
| 410 | 413 |
var localFile = fileAgent.GetFileInfo(relativePath); |
| 414 |
if (objectInfo.Content_Type == @"application/directory" && localFile is DirectoryInfo) |
|
| 415 |
continue; |
|
| 411 | 416 |
var state = FileState.FindByFilePath(localFile.FullName); |
| 412 | 417 |
//Common files should be checked on a per-case basis to detect differences, which is newer |
| 413 | 418 |
|
| ... | ... | |
| 756 | 761 |
var cloudFile = action.CloudFile; |
| 757 | 762 |
var account = cloudFile.Account ?? accountInfo.UserName; |
| 758 | 763 |
|
| 759 |
var client = new CloudFilesClient(accountInfo); |
|
| 764 |
var client = new CloudFilesClient(accountInfo);
|
|
| 760 | 765 |
//Even if GetObjectInfo times out, we can proceed with the upload |
| 761 | 766 |
var info = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name); |
| 762 |
var cloudHash = info.Hash.ToLower(); |
|
| 763 | 767 |
|
| 764 |
var hash = action.LocalHash.Value; |
|
| 765 |
var topHash = action.TopHash.Value; |
|
| 768 |
//If this is a read-only file, do not upload changes |
|
| 769 |
if (info.AllowedTo == "read") |
|
| 770 |
return; |
|
| 766 | 771 |
|
| 767 |
//If the file hashes match, abort the upload |
|
| 768 |
if (hash == cloudHash || topHash == cloudHash) |
|
| 772 |
//WRONG: If this is a directory, there is no hash to check. ???? |
|
| 773 |
//TODO: Check how a directory hash is calculated |
|
| 774 |
if (fileInfo is DirectoryInfo) |
|
| 769 | 775 |
{
|
| 770 |
//but store any metadata changes
|
|
| 771 |
StatusKeeper.StoreInfo(fullFileName, info);
|
|
| 772 |
Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
|
|
| 773 |
return;
|
|
| 776 |
//If the directory doesn't exist the Hash property will be empty
|
|
| 777 |
if (String.IsNullOrWhiteSpace(info.Hash))
|
|
| 778 |
//Go on and create the directory
|
|
| 779 |
client.PutObject(account, cloudFile.Container, cloudFile.Name, fileInfo.FullName, String.Empty, "application/directory");
|
|
| 774 | 780 |
} |
| 781 |
else |
|
| 782 |
{
|
|
| 775 | 783 |
|
| 776 |
if (info.AllowedTo == "read") |
|
| 777 |
return; |
|
| 784 |
var cloudHash = info.Hash.ToLower(); |
|
| 778 | 785 |
|
| 779 |
//Mark the file as modified while we upload it |
|
| 780 |
StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified); |
|
| 781 |
//And then upload it |
|
| 786 |
var hash = action.LocalHash.Value; |
|
| 787 |
var topHash = action.TopHash.Value; |
|
| 788 |
|
|
| 789 |
//If the file hashes match, abort the upload |
|
| 790 |
if (hash == cloudHash || topHash == cloudHash) |
|
| 791 |
{
|
|
| 792 |
//but store any metadata changes |
|
| 793 |
StatusKeeper.StoreInfo(fullFileName, info); |
|
| 794 |
Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
|
|
| 795 |
return; |
|
| 796 |
} |
|
| 782 | 797 |
|
| 783 |
//Upload even small files using the Hashmap. The server may already containt |
|
| 784 |
//the relevant folder |
|
| 785 | 798 |
|
| 786 |
//First, calculate the tree hash
|
|
| 787 |
var treeHash = await Signature.CalculateTreeHashAsync(fileInfo.FullName, accountInfo.BlockSize,
|
|
| 788 |
accountInfo.BlockHash);
|
|
| 799 |
//Mark the file as modified while we upload it
|
|
| 800 |
StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
|
|
| 801 |
//And then upload it
|
|
| 789 | 802 |
|
| 790 |
await UploadWithHashMap(accountInfo, cloudFile, fileInfo, cloudFile.Name, treeHash); |
|
| 803 |
//Upload even small files using the Hashmap. The server may already contain |
|
| 804 |
//the relevant block |
|
| 791 | 805 |
|
| 806 |
//First, calculate the tree hash |
|
| 807 |
var treeHash = await Signature.CalculateTreeHashAsync(fileInfo.FullName, accountInfo.BlockSize, |
|
| 808 |
accountInfo.BlockHash); |
|
| 809 |
|
|
| 810 |
await UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash); |
|
| 811 |
} |
|
| 792 | 812 |
//If everything succeeds, change the file and overlay status to normal |
| 793 | 813 |
StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal); |
| 794 | 814 |
} |
Also available in: Unified diff