X-Git-Url: https://code.grnet.gr/git/pithos-ms-client/blobdiff_plain/a60e5887334ff99df9ccb0f9839b74a23968bc46..d7288179638247cb10a9abec6f85503d1c72315f:/trunk/Pithos.Core/Agents/Downloader.cs diff --git a/trunk/Pithos.Core/Agents/Downloader.cs b/trunk/Pithos.Core/Agents/Downloader.cs index 75f1844..584141a 100644 --- a/trunk/Pithos.Core/Agents/Downloader.cs +++ b/trunk/Pithos.Core/Agents/Downloader.cs @@ -1,9 +1,7 @@ using System; -using System.Collections.Generic; using System.ComponentModel.Composition; using System.Diagnostics.Contracts; using System.IO; -using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -21,6 +19,8 @@ namespace Pithos.Core.Agents [Import] private IStatusKeeper StatusKeeper { get; set; } + [Import] + private IPithosSettings Settings { get; set; } public IStatusNotification StatusNotification { get; set; } @@ -35,7 +35,7 @@ namespace Pithos.Core.Agents //Download a file. - public async Task DownloadCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile, string filePath,CancellationToken cancellationToken) + public async Task DownloadCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile, string filePath,CancellationToken cancellationToken) { if (accountInfo == null) throw new ArgumentNullException("accountInfo"); @@ -43,7 +43,9 @@ namespace Pithos.Core.Agents throw new ArgumentNullException("cloudFile"); if (String.IsNullOrWhiteSpace(cloudFile.Account)) throw new ArgumentNullException("cloudFile"); - if (String.IsNullOrWhiteSpace(cloudFile.Container)) + if (cloudFile.Container==null) + throw new ArgumentNullException("cloudFile"); + if (cloudFile.Container.IsAbsoluteUri) throw new ArgumentNullException("cloudFile"); if (String.IsNullOrWhiteSpace(filePath)) throw new ArgumentNullException("filePath"); @@ -54,40 +56,43 @@ namespace Pithos.Core.Agents { // var cancellationToken=_cts.Token;// .ThrowIfCancellationRequested(); + //The file's treehash after download completes. For directories, the treehash is always the empty hash + var finalHash = TreeHash.Empty; + if (await WaitOrAbort(accountInfo,cloudFile, cancellationToken).ConfigureAwait(false)) - return; + return finalHash; var fileName = Path.GetFileName(filePath); - var progress = new Progress(d => - StatusNotification.Notify(new StatusNotification(String.Format("Hashing for Download {0} of {1}", d, fileName)))); + var info = FileInfoExtensions.FromPath(filePath).WithProperCapitalization(); TreeHash localTreeHash; - + using (StatusNotification.GetNotifier("Hashing for Download {0}", "Hashed for Download {0}", fileName)) { + var state = StatusKeeper.GetStateByFilePath(filePath); + var progress = new Progress(d => + StatusNotification.Notify(new StatusNotification(String.Format("Hashing for Download {0} of {1}", d, fileName)))); - localTreeHash = Signature.CalculateTreeHashAsync(filePath, - accountInfo.BlockSize, - accountInfo.BlockHash, 1,progress); + localTreeHash = StatusAgent.CalculateTreeHash(info, accountInfo, state, Settings.HashingParallelism, cancellationToken, progress); } - - var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath); - var relativeUrl = new Uri(cloudFile.Name, UriKind.Relative); + + var localPath = info.FullName; + var relativeUrl = cloudFile.Name; var url = relativeUrl.ToString(); - if (cloudFile.Name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase)) - return; + if (url.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase)) + return finalHash; if (!Selectives.IsSelected(accountInfo,cloudFile)) - return; + return finalHash; //Are we already downloading or uploading the file? using (var gate = NetworkGate.Acquire(localPath, NetworkOperation.Downloading)) { if (gate.Failed) - return; + return finalHash; var client = new CloudFilesClient(accountInfo); var account = cloudFile.Account; @@ -117,11 +122,11 @@ namespace Pithos.Core.Agents } else { - var isChanged = IsObjectChanged(cloudFile, localPath); + var isChanged = IsObjectChanged(cloudFile, localPath,localTreeHash); if (isChanged) { //Retrieve the hashmap from the server - var serverHash = await client.GetHashMap(account, container, url).ConfigureAwait(false); + var serverHash = await client.GetHashMap(account, container, relativeUrl).ConfigureAwait(false); //If it's a small file if (serverHash.Hashes.Count == 1) //Download it in one go @@ -138,13 +143,18 @@ namespace Pithos.Core.Agents var attributes = File.GetAttributes(localPath); File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly); } + + //Once download completes, the final hash will be equal to the server hash + finalHash = serverHash; + } } //Now we can store the object's metadata without worrying about ghost status entries - StatusKeeper.StoreInfo(localPath, cloudFile); + StatusKeeper.StoreInfo(localPath, cloudFile,finalHash); } + return finalHash; } } @@ -184,9 +194,8 @@ namespace Pithos.Core.Agents var fileName = Path.GetFileName(localPath); var progress = new Progress(d => StatusNotification.Notify(new StatusNotification(String.Format("Hashing for Download {0} of {1}", d, fileName)))); - - //TODO: Should pass cancellation token here - var treeHash = localTreeHash ?? Signature.CalculateTreeHashAsync(localPath, (int)serverHash.BlockSize, serverHash.BlockHash, 2,progress); + + var treeHash = localTreeHash ?? Signature.CalculateTreeHashAsync(localPath, (int)serverHash.BlockSize, serverHash.BlockHash, Settings.HashingParallelism,cancellationToken,progress); //And compare it with the server's hash var upHashes = serverHash.GetHashesAsStrings(); @@ -223,7 +232,7 @@ namespace Pithos.Core.Agents if (i < upHashes.Length - 1) end = ((i + 1) * serverHash.BlockSize); - //TODO: Pass token here + //Download the missing block byte[] block = await client.GetBlock(cloudFile.Account, cloudFile.Container, relativeUrl, start, end, cancellationToken).ConfigureAwait(false); @@ -245,6 +254,7 @@ namespace Pithos.Core.Agents StatusNotification.NotifyChangedFile(localPath); Log.InfoFormat("[BLOCK GET] COMPLETE {0}", localPath); + } //Download a small file with a single GET operation @@ -282,10 +292,8 @@ namespace Pithos.Core.Agents if (!Directory.Exists(tempFolder)) Directory.CreateDirectory(tempFolder); - //TODO: Should pass the token here - //Download the object to the temporary location - await client.GetObject(cloudFile.Account, cloudFile.Container, relativeUrl.ToString(), tempPath, cancellationToken).ConfigureAwait(false); + await client.GetObject(cloudFile.Account, cloudFile.Container, relativeUrl, tempPath, cancellationToken).ConfigureAwait(false); //Create the local folder if it doesn't exist (necessary for shared objects) var localFolder = Path.GetDirectoryName(localPath); @@ -327,7 +335,7 @@ namespace Pithos.Core.Agents : new ProgressNotification(fileName, "Downloading", block, blockPercentage, totalBlocks, fileSize)); } - private bool IsObjectChanged(ObjectInfo cloudFile, string localPath) + private bool IsObjectChanged(ObjectInfo cloudFile, string localPath,TreeHash localTreeHash) { //If the target is a directory, there are no changes to download if (Directory.Exists(localPath)) @@ -340,10 +348,9 @@ namespace Pithos.Core.Agents if (localState == null) return true; - var info = new FileInfo(localPath); - var etag = info.ComputeShortHash(StatusNotification); + var localHash= localTreeHash.TopHash.ToHashString(); //If the file is different from the stored state, we have a change - if (localState.ETag != etag) + if (localState.Checksum != localHash) return true; //If the top hashes differ, we have a change return (localState.Checksum != cloudFile.X_Object_Hash);