X-Git-Url: https://code.grnet.gr/git/pithos-ms-client/blobdiff_plain/b666b39aaca5965ba58b484f1042c6a952f210f4..fbe71a057386c4d4f0b65d51dd25289b325a3847:/trunk/Pithos.Core/Agents/NetworkAgent.cs diff --git a/trunk/Pithos.Core/Agents/NetworkAgent.cs b/trunk/Pithos.Core/Agents/NetworkAgent.cs index d051dbf..6431ffe 100644 --- a/trunk/Pithos.Core/Agents/NetworkAgent.cs +++ b/trunk/Pithos.Core/Agents/NetworkAgent.cs @@ -40,11 +40,7 @@ */ #endregion -//TODO: Now there is a UUID tag. This can be used for renames/moves - - using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel.Composition; using System.Diagnostics; @@ -52,9 +48,9 @@ using System.Diagnostics.Contracts; using System.IO; using System.Linq; using System.Net; +using System.Reflection; using System.Threading; using System.Threading.Tasks; -using System.Threading.Tasks.Dataflow; using Castle.ActiveRecord; using Pithos.Interfaces; using Pithos.Network; @@ -62,46 +58,117 @@ using log4net; namespace Pithos.Core.Agents { - //TODO: Ensure all network operations use exact casing. Pithos is case sensitive [Export] public class NetworkAgent { + private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + private Agent _agent; [System.ComponentModel.Composition.Import] - private DeleteAgent _deleteAgent=new DeleteAgent(); + private DeleteAgent DeleteAgent { get; set; } [System.ComponentModel.Composition.Import] public IStatusKeeper StatusKeeper { get; set; } - - public IStatusNotification StatusNotification { get; set; } - private static readonly ILog Log = LogManager.GetLogger("NetworkAgent"); + private IStatusNotification _statusNotification; + public IStatusNotification StatusNotification + { + get { return _statusNotification; } + set + { + _statusNotification = value; + DeleteAgent.StatusNotification = value; + Uploader.StatusNotification = value; + Downloader.StatusNotification = value; + } + } - private readonly ConcurrentBag _accounts = new ConcurrentBag(); [System.ComponentModel.Composition.Import] public IPithosSettings Settings { get; set; } + private Uploader _uploader; + + [System.ComponentModel.Composition.Import] + public Uploader Uploader + { + get { return _uploader; } + set + { + _uploader = value; + _uploader.UnpauseEvent = _unPauseEvent; + } + } + + private Downloader _downloader; + + [System.ComponentModel.Composition.Import] + public Downloader Downloader + { + get { return _downloader; } + set + { + _downloader = value; + _downloader.UnpauseEvent = _unPauseEvent; + } + } + + [System.ComponentModel.Composition.Import] + public Selectives Selectives { get; set; } + //The Proceed signals the poll agent that it can proceed with polling. //Essentially it stops the poll agent to give priority to the network agent //Initially the event is signalled because we don't need to pause private readonly AsyncManualResetEvent _proceedEvent = new AsyncManualResetEvent(true); + private Agents.Selectives _selectives; + private bool _pause; public AsyncManualResetEvent ProceedEvent { get { return _proceedEvent; } } + private readonly AsyncManualResetEvent _unPauseEvent = new AsyncManualResetEvent(true); + + private CancellationTokenSource _currentOperationCancellation=new CancellationTokenSource(); + + public void CancelCurrentOperation() + { + //What does it mean to cancel the current upload/download? + //Obviously, the current operation will be cancelled by throwing + //a cancellation exception. + // + //The default behavior is to retry any operations that throw. + //Obviously this is not what we want in this situation. + //The cancelled operation should NOT bea retried. + // + //This can be done by catching the cancellation exception + //and avoiding the retry. + // + + //Have to reset the cancellation source - it is not possible to reset the source + //Have to prevent a case where an operation requests a token from the old source + var oldSource = Interlocked.Exchange(ref _currentOperationCancellation, new CancellationTokenSource()); + oldSource.Cancel(); + + } public void Start() { + if (_agent != null) + return; + + if (Log.IsDebugEnabled) + Log.Debug("Starting Network Agent"); + _agent = Agent.Start(inbox => { Action loop = null; loop = () => { - _deleteAgent.ProceedEvent.Wait(); + DeleteAgent.ProceedEvent.Wait(); + _unPauseEvent.Wait(); var message = inbox.Receive(); var process=message.Then(Process,inbox.CancellationToken); inbox.LoopAsync(process, loop); @@ -122,29 +189,29 @@ namespace Pithos.Core.Agents - using (log4net.ThreadContext.Stacks["NETWORK"].Push("PROCESS")) + using (ThreadContext.Stacks["Operation"].Push(action.ToString())) { - Log.InfoFormat("[ACTION] Start Processing {0}", action); var cloudFile = action.CloudFile; var downloadPath = action.GetDownloadPath(); try { + StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Processing"); _proceedEvent.Reset(); - UpdateStatus(PithosStatus.Syncing); + var accountInfo = action.AccountInfo; if (action.Action == CloudActionType.DeleteCloud) { //Redirect deletes to the delete agent - _deleteAgent.Post((CloudDeleteAction)action); + DeleteAgent.Post((CloudDeleteAction)action); } - if (_deleteAgent.IsDeletedFile(action)) + if (DeleteAgent.IsDeletedFile(action)) { //Clear the status of already deleted files to avoid reprocessing if (action.LocalFile != null) - this.StatusKeeper.ClearFileStatus(action.LocalFile.FullName); + StatusKeeper.ClearFileStatus(action.LocalFile.FullName); } else { @@ -152,10 +219,12 @@ namespace Pithos.Core.Agents { case CloudActionType.UploadUnconditional: //Abort if the file was deleted before we reached this point - await UploadCloudFile(action); + var uploadAction = (CloudUploadAction) action; + ProcessChildUploads(uploadAction); + await Uploader.UploadCloudFile(uploadAction ,CurrentOperationCancelToken); break; case CloudActionType.DownloadUnconditional: - await DownloadCloudFile(accountInfo, cloudFile, downloadPath); + await Downloader.DownloadCloudFile(accountInfo, cloudFile, downloadPath, CurrentOperationCancelToken); break; case CloudActionType.RenameCloud: var moveAction = (CloudMoveAction)action; @@ -167,7 +236,7 @@ namespace Pithos.Core.Agents case CloudActionType.MustSynch: if (!File.Exists(downloadPath) && !Directory.Exists(downloadPath)) { - await DownloadCloudFile(accountInfo, cloudFile, downloadPath); + await Downloader.DownloadCloudFile(accountInfo, cloudFile, downloadPath, CurrentOperationCancelToken); } else { @@ -176,16 +245,27 @@ namespace Pithos.Core.Agents break; } } - Log.InfoFormat("[ACTION] End Processing {0}:{1}->{2}", action.Action, action.LocalFile, + Log.InfoFormat("End Processing {0}:{1}->{2}", action.Action, action.LocalFile, action.CloudFile.Name); } +/* catch (WebException exc) - { + { Log.ErrorFormat("[WEB ERROR] {0} : {1} -> {2} due to exception\r\n{3}", action.Action, action.LocalFile, action.CloudFile, exc); + + + //Actions that resulted in server errors should be retried + var response = exc.Response as HttpWebResponse; + if (response != null && response.StatusCode >= HttpStatusCode.InternalServerError) + { + _agent.Post(action); + Log.WarnFormat("[REQUEUE] {0} : {1} -> {2}", action.Action, action.LocalFile, action.CloudFile); + } } - catch (OperationCanceledException) - { - throw; +*/ + catch (OperationCanceledException ex) + { + Log.WarnFormat("Cancelling [{0}]",ex); } catch (DirectoryNotFoundException) { @@ -212,16 +292,49 @@ namespace Pithos.Core.Agents { if (_agent.IsEmpty) _proceedEvent.Set(); - UpdateStatus(PithosStatus.InSynch); + UpdateStatus(PithosStatus.LocalComplete); } } } + private void ProcessChildUploads(CloudUploadAction uploadAction) + { + if (!uploadAction.IsCreation || !(uploadAction.LocalFile is DirectoryInfo)) + return; + + var dirInfo = uploadAction.LocalFile as DirectoryInfo; + + var account = uploadAction.AccountInfo; + var actions = from file in dirInfo.EnumerateFiles("*", SearchOption.AllDirectories) + select + new CloudUploadAction(account, file, null, account.BlockSize, account.BlockHash, + uploadAction, true); + foreach (var action in actions) + { + var state=StatusKeeper.GetStateByFilePath(action.LocalFile.FullName); + if (state!=null) + state.Delete(); + //StatusKeeper.SetFileState(action.LocalFile.FullName,FileStatus.Created,FileOverlayStatus.Normal,String.Empty); + state=FileState.CreateFor(action.LocalFile); + //StatusKeeper.SetFileStatus(); + state.FileStatus = FileStatus.Created; + state.OverlayStatus=FileOverlayStatus.Normal; + state.Create(); + action.FileState = state; + Post(action); + } + } + + private CancellationToken CurrentOperationCancelToken + { + get { return _currentOperationCancellation.Token; } + } + private void UpdateStatus(PithosStatus status) { - StatusKeeper.SetPithosStatus(status); - StatusNotification.Notify(new Notification()); + StatusNotification.SetPithosStatus(status); + //StatusNotification.Notify(new Notification()); } private void RenameLocalFile(AccountInfo accountInfo, CloudAction action) @@ -237,20 +350,22 @@ namespace Pithos.Core.Agents if (action.CloudFile == null) throw new ArgumentException("The action's cloud file is not specified", "action"); Contract.EndContractBlock(); + using (ThreadContext.Stacks["Operation"].Push("RenameLocalFile")) + { - //We assume that the local file already exists, otherwise the poll agent - //would have issued a download request - - var currentInfo = action.CloudFile; - var previousInfo = action.CloudFile.Previous; - var fileAgent = FileAgent.GetFileAgent(accountInfo); + //We assume that the local file already exists, otherwise the poll agent + //would have issued a download request - var previousRelativepath = previousInfo.RelativeUrlToFilePath(accountInfo.UserName); - var previousFile = fileAgent.GetFileSystemInfo(previousRelativepath); + var currentInfo = action.CloudFile; + var previousInfo = action.CloudFile.Previous; + var fileAgent = FileAgent.GetFileAgent(accountInfo); - //In every case we need to move the local file first - MoveLocalFile(accountInfo, previousFile, fileAgent, currentInfo); + var previousRelativepath = previousInfo.RelativeUrlToFilePath(accountInfo.UserName); + var previousFile = fileAgent.GetFileSystemInfo(previousRelativepath); + //In every case we need to move the local file first + MoveLocalFile(accountInfo, previousFile, fileAgent, currentInfo); + } } private void MoveLocalFile(AccountInfo accountInfo, FileSystemInfo previousFile, FileAgent fileAgent, @@ -264,8 +379,8 @@ namespace Pithos.Core.Agents FileInfoExtensions.GetProperFilePathCapitalization(previousFile.FullName): FileInfoExtensions.GetProperDirectoryCapitalization(previousFile.FullName); - using (var gateOld = NetworkGate.Acquire(previousFullPath, NetworkOperation.Renaming)) - using (var gateNew = NetworkGate.Acquire(newPath,NetworkOperation.Renaming)) + using (NetworkGate.Acquire(previousFullPath, NetworkOperation.Renaming)) + using (NetworkGate.Acquire(newPath,NetworkOperation.Renaming)) using (new SessionScope(FlushAction.Auto)) { if (isFile) @@ -277,7 +392,7 @@ namespace Pithos.Core.Agents var state = StatusKeeper.GetStateByFilePath(previousFullPath); state.FilePath = newPath; state.SaveCopy(); - StatusKeeper.SetFileState(previousFullPath,FileStatus.Deleted,FileOverlayStatus.Deleted); + StatusKeeper.SetFileState(previousFullPath,FileStatus.Deleted,FileOverlayStatus.Deleted, "Deleted"); } } @@ -294,51 +409,66 @@ namespace Pithos.Core.Agents if (action.CloudFile== null) throw new ArgumentException("The action's cloud file is not specified", "action"); Contract.EndContractBlock(); + using (ThreadContext.Stacks["Operation"].Push("SyncFiles")) + { - var localFile = action.LocalFile; - var cloudFile = action.CloudFile; - var downloadPath=action.LocalFile.GetProperCapitalization(); + //var localFile = action.LocalFile; + var cloudFile = action.CloudFile; + var downloadPath = action.LocalFile.GetProperCapitalization(); - var cloudHash = cloudFile.Hash.ToLower(); - var previousCloudHash = cloudFile.PreviousHash.ToLower(); - var localHash = action.LocalHash.Value.ToLower(); - var topHash = action.TopHash.Value.ToLower(); + var cloudHash = cloudFile.Hash.ToLower(); + var previousCloudHash = cloudFile.PreviousHash == null?null: cloudFile.PreviousHash.ToLower(); + var localHash = action.TreeHash.Value.TopHash.ToHashString();// LocalHash.Value.ToLower(); + //var topHash = action.TopHash.Value.ToLower(); - //At this point we know that an object has changed on the server and that a local - //file already exists. We need to decide whether the file has only changed on - //the server or there is a conflicting change on the client. - // - - //Not enough to compare only the local hashes (MD5), also have to compare the tophashes - //If any of the hashes match, we are done - if ((cloudHash == localHash || cloudHash == topHash)) - { - Log.InfoFormat("Skipping {0}, hashes match",downloadPath); - return; - } + if(cloudFile.IsDirectory && action.LocalFile is DirectoryInfo) + { + Log.InfoFormat("Skipping folder {0} , exists in server", downloadPath); + return; + } - //The hashes DON'T match. We need to sync + //At this point we know that an object has changed on the server and that a local + //file already exists. We need to decide whether the file has only changed on + //the server or there is a conflicting change on the client. + // - // If the previous tophash matches the local tophash, the file was only changed on the server. - if (localHash == previousCloudHash) - { - await DownloadCloudFile(accountInfo, cloudFile, downloadPath); - } - else - { - //If the previous and local hash don't match, there was a local conflict - //that was not uploaded to the server. We have a conflict - ReportConflict(downloadPath); + //If the hashes match, we are done + if (cloudFile != ObjectInfo.Empty && cloudHash == localHash) + { + Log.InfoFormat("Skipping {0}, hashes match", downloadPath); + return; + } + + //If the local and remote files have 0 length their hashes will not match + if (!cloudFile.IsDirectory && cloudFile.Bytes==0 && action.LocalFile is FileInfo && (action.LocalFile as FileInfo).Length==0 ) + { + Log.InfoFormat("Skipping {0}, files are empty", downloadPath); + return; + } + + //The hashes DON'T match. We need to sync + + // If the previous tophash matches the local tophash, the file was only changed on the server. + if (localHash == previousCloudHash) + { + await Downloader.DownloadCloudFile(accountInfo, cloudFile, downloadPath, CurrentOperationCancelToken); + } + else + { + //If the previous and local hash don't match, there was a local conflict + //that was not uploaded to the server. We have a conflict + ReportConflictForMismatch(downloadPath); + } } } - private void ReportConflict(string downloadPath) + private void ReportConflictForMismatch(string downloadPath) { if (String.IsNullOrWhiteSpace(downloadPath)) throw new ArgumentNullException("downloadPath"); Contract.EndContractBlock(); - StatusKeeper.SetFileOverlayStatus(downloadPath, FileOverlayStatus.Conflict); + StatusKeeper.SetFileState(downloadPath,FileStatus.Conflict, FileOverlayStatus.Conflict,"File changed at the server"); UpdateStatus(PithosStatus.HasConflicts); var message = String.Format("Conflict detected for file {0}", downloadPath); Log.Warn(message); @@ -353,32 +483,10 @@ namespace Pithos.Core.Agents throw new ArgumentException("The CloudAction.AccountInfo is empty","cloudAction"); Contract.EndContractBlock(); - _deleteAgent.ProceedEvent.Wait(); - - //If the action targets a local file, add a treehash calculation - if (!(cloudAction is CloudDeleteAction) && cloudAction.LocalFile as FileInfo != null) - { - var accountInfo = cloudAction.AccountInfo; - var localFile = (FileInfo) cloudAction.LocalFile; - if (localFile.Length > accountInfo.BlockSize) - cloudAction.TopHash = - new Lazy(() => Signature.CalculateTreeHashAsync(localFile, - accountInfo.BlockSize, - accountInfo.BlockHash, Settings.HashingParallelism).Result - .TopHash.ToHashString()); - else - { - cloudAction.TopHash = new Lazy(() => cloudAction.LocalHash.Value); - } - } - else - { - //The hash for a directory is the empty string - cloudAction.TopHash = new Lazy(() => String.Empty); - } + DeleteAgent.ProceedEvent.Wait(); if (cloudAction is CloudDeleteAction) - _deleteAgent.Post((CloudDeleteAction)cloudAction); + DeleteAgent.Post((CloudDeleteAction)cloudAction); else _agent.Post(cloudAction); } @@ -391,20 +499,30 @@ namespace Pithos.Core.Agents public Task GetDeleteAwaiter() { - return _deleteAgent.ProceedEvent.WaitAsync(); + return DeleteAgent.ProceedEvent.WaitAsync(); } public CancellationToken CancellationToken { get { return _agent.CancellationToken; } } - private static FileAgent GetFileAgent(AccountInfo accountInfo) + public bool Pause { - return AgentLocator.Get(accountInfo.AccountPath); + get { + return _pause; + } + set { + _pause = value; + if (_pause) + _unPauseEvent.Reset(); + else + { + _unPauseEvent.Set(); + } + } } - private void RenameCloudFile(AccountInfo accountInfo,CloudMoveAction action) { if (accountInfo==null) @@ -420,443 +538,44 @@ namespace Pithos.Core.Agents if (action.OldCloudFile==null) throw new ArgumentException("OldCloudFile","action"); Contract.EndContractBlock(); - - - var newFilePath = action.LocalFile.FullName; - - //How do we handle concurrent renames and deletes/uploads/downloads? - //* A conflicting upload means that a file was renamed before it had a chance to finish uploading - // This should never happen as the network agent executes only one action at a time - //* A conflicting download means that the file was modified on the cloud. While we can go on and complete - // the rename, there may be a problem if the file is downloaded in blocks, as subsequent block requests for the - // same name will fail. - // This should never happen as the network agent executes only one action at a time. - //* A conflicting delete can happen if the rename was followed by a delete action that didn't have the chance - // to remove the rename from the queue. - // We can probably ignore this case. It will result in an error which should be ignored - - - //The local file is already renamed - StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Modified); - - - var account = action.CloudFile.Account ?? accountInfo.UserName; - var container = action.CloudFile.Container; - - var client = new CloudFilesClient(accountInfo); - //TODO: What code is returned when the source file doesn't exist? - client.MoveObject(account, container, action.OldCloudFile.Name, container, action.CloudFile.Name); - - StatusKeeper.SetFileStatus(newFilePath, FileStatus.Unchanged); - StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Normal); - NativeMethods.RaiseChangeNotification(newFilePath); - } - - //Download a file. - private async Task DownloadCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile , string filePath) - { - if (accountInfo == null) - throw new ArgumentNullException("accountInfo"); - if (cloudFile == null) - throw new ArgumentNullException("cloudFile"); - if (String.IsNullOrWhiteSpace(cloudFile.Account)) - throw new ArgumentNullException("cloudFile"); - if (String.IsNullOrWhiteSpace(cloudFile.Container)) - throw new ArgumentNullException("cloudFile"); - if (String.IsNullOrWhiteSpace(filePath)) - throw new ArgumentNullException("filePath"); - if (!Path.IsPathRooted(filePath)) - throw new ArgumentException("The filePath must be rooted", "filePath"); - Contract.EndContractBlock(); - - - var localPath = Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath); - var relativeUrl = new Uri(cloudFile.Name, UriKind.Relative); - var url = relativeUrl.ToString(); - if (cloudFile.Name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase)) - return; - - - //Are we already downloading or uploading the file? - using (var gate=NetworkGate.Acquire(localPath, NetworkOperation.Downloading)) - { - if (gate.Failed) - return; - - var client = new CloudFilesClient(accountInfo); - var account = cloudFile.Account; - var container = cloudFile.Container; - - if (cloudFile.Content_Type == @"application/directory") - { - if (!Directory.Exists(localPath)) - Directory.CreateDirectory(localPath); - } - else - { - //Retrieve the hashmap from the server - var serverHash = await client.GetHashMap(account, container, url); - //If it's a small file - if (serverHash.Hashes.Count == 1) - //Download it in one go - await - DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath, serverHash); - //Otherwise download it block by block - else - await DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath, serverHash); - - if (cloudFile.AllowedTo == "read") - { - var attributes = File.GetAttributes(localPath); - File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly); - } - } - - //Now we can store the object's metadata without worrying about ghost status entries - StatusKeeper.StoreInfo(localPath, cloudFile); - - } - } - - //Download a small file with a single GET operation - private async Task DownloadEntireFileAsync(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath,TreeHash serverHash) - { - if (client == null) - throw new ArgumentNullException("client"); - if (cloudFile==null) - throw new ArgumentNullException("cloudFile"); - if (relativeUrl == null) - throw new ArgumentNullException("relativeUrl"); - if (String.IsNullOrWhiteSpace(filePath)) - throw new ArgumentNullException("filePath"); - if (!Path.IsPathRooted(filePath)) - throw new ArgumentException("The localPath must be rooted", "filePath"); - Contract.EndContractBlock(); - - var localPath = Pithos.Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath); - //If the file already exists - if (File.Exists(localPath)) + using (ThreadContext.Stacks["Operation"].Push("RenameCloudFile")) { - //First check with MD5 as this is a small file - var localMD5 = Signature.CalculateMD5(localPath); - var cloudHash=serverHash.TopHash.ToHashString(); - if (localMD5==cloudHash) - return; - //Then check with a treehash - var localTreeHash = Signature.CalculateTreeHash(localPath, serverHash.BlockSize, serverHash.BlockHash); - var localHash = localTreeHash.TopHash.ToHashString(); - if (localHash==cloudHash) - return; - } - StatusNotification.Notify(new CloudNotification { Data = cloudFile }); - - var fileAgent = GetFileAgent(accountInfo); - //Calculate the relative file path for the new file - var relativePath = relativeUrl.RelativeUriToFilePath(); - //The file will be stored in a temporary location while downloading with an extension .download - var tempPath = Path.Combine(fileAgent.CachePath, relativePath + ".download"); - //Make sure the target folder exists. DownloadFileTask will not create the folder - var tempFolder = Path.GetDirectoryName(tempPath); - if (!Directory.Exists(tempFolder)) - Directory.CreateDirectory(tempFolder); - - //Download the object to the temporary location - await client.GetObject(cloudFile.Account, cloudFile.Container, relativeUrl.ToString(), tempPath); - - //Create the local folder if it doesn't exist (necessary for shared objects) - var localFolder = Path.GetDirectoryName(localPath); - if (!Directory.Exists(localFolder)) - Directory.CreateDirectory(localFolder); - //And move it to its actual location once downloading is finished - if (File.Exists(localPath)) - File.Replace(tempPath,localPath,null,true); - else - File.Move(tempPath,localPath); - //Notify listeners that a local file has changed - StatusNotification.NotifyChangedFile(localPath); - - - } - - //Download a file asynchronously using blocks - public async Task DownloadWithBlocks(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath, TreeHash serverHash) - { - if (client == null) - throw new ArgumentNullException("client"); - if (cloudFile == null) - throw new ArgumentNullException("cloudFile"); - if (relativeUrl == null) - throw new ArgumentNullException("relativeUrl"); - if (String.IsNullOrWhiteSpace(filePath)) - throw new ArgumentNullException("filePath"); - if (!Path.IsPathRooted(filePath)) - throw new ArgumentException("The filePath must be rooted", "filePath"); - if (serverHash == null) - throw new ArgumentNullException("serverHash"); - Contract.EndContractBlock(); - - var fileAgent = GetFileAgent(accountInfo); - var localPath = Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath); - - //Calculate the relative file path for the new file - var relativePath = relativeUrl.RelativeUriToFilePath(); - var blockUpdater = new BlockUpdater(fileAgent.CachePath, localPath, relativePath, serverHash); - - - - //Calculate the file's treehash - var treeHash = await Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize, serverHash.BlockHash, 2); - - //And compare it with the server's hash - var upHashes = serverHash.GetHashesAsStrings(); - var localHashes = treeHash.HashDictionary; - for (int i = 0; i < upHashes.Length; i++) - { - //For every non-matching hash - var upHash = upHashes[i]; - if (!localHashes.ContainsKey(upHash)) - { - StatusNotification.Notify(new CloudNotification { Data = cloudFile }); - - if (blockUpdater.UseOrphan(i, upHash)) - { - Log.InfoFormat("[BLOCK GET] ORPHAN FOUND for {0} of {1} for {2}", i, upHashes.Length, localPath); - continue; - } - Log.InfoFormat("[BLOCK GET] START {0} of {1} for {2}", i, upHashes.Length, localPath); - var start = i*serverHash.BlockSize; - //To download the last block just pass a null for the end of the range - long? end = null; - if (i < upHashes.Length - 1 ) - end= ((i + 1)*serverHash.BlockSize) ; - - //Download the missing block - var block = await client.GetBlock(cloudFile.Account, cloudFile.Container, relativeUrl, start, end); - - //and store it - blockUpdater.StoreBlock(i, block); - - - Log.InfoFormat("[BLOCK GET] FINISH {0} of {1} for {2}", i, upHashes.Length, localPath); - } - } - - //Want to avoid notifications if no changes were made - var hasChanges = blockUpdater.HasBlocks; - blockUpdater.Commit(); - - if (hasChanges) - //Notify listeners that a local file has changed - StatusNotification.NotifyChangedFile(localPath); - - Log.InfoFormat("[BLOCK GET] COMPLETE {0}", localPath); - } - - - private async Task UploadCloudFile(CloudAction action) - { - if (action == null) - throw new ArgumentNullException("action"); - Contract.EndContractBlock(); - - try - { - var accountInfo = action.AccountInfo; - - var fileInfo = action.LocalFile; - - if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)) - return; - - var relativePath = fileInfo.AsRelativeTo(accountInfo.AccountPath); - if (relativePath.StartsWith(FolderConstants.OthersFolder)) - { - var parts = relativePath.Split('\\'); - var accountName = parts[1]; - var oldName = accountInfo.UserName; - var absoluteUri = accountInfo.StorageUri.AbsoluteUri; - var nameIndex = absoluteUri.IndexOf(oldName); - var root = absoluteUri.Substring(0, nameIndex); - - accountInfo = new AccountInfo - { - UserName = accountName, - AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]), - StorageUri = new Uri(root + accountName), - BlockHash = accountInfo.BlockHash, - BlockSize = accountInfo.BlockSize, - Token = accountInfo.Token - }; - } - - - var fullFileName = fileInfo.GetProperCapitalization(); - using (var gate = NetworkGate.Acquire(fullFileName, NetworkOperation.Uploading)) - { - //Abort if the file is already being uploaded or downloaded - if (gate.Failed) - return; - - var cloudFile = action.CloudFile; - var account = cloudFile.Account ?? accountInfo.UserName; - - var client = new CloudFilesClient(accountInfo); - //Even if GetObjectInfo times out, we can proceed with the upload - var info = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name); - - //If this is a read-only file, do not upload changes - if (info.AllowedTo == "read") - return; - - //TODO: Check how a directory hash is calculated -> All dirs seem to have the same hash - if (fileInfo is DirectoryInfo) - { - //If the directory doesn't exist the Hash property will be empty - if (String.IsNullOrWhiteSpace(info.Hash)) - //Go on and create the directory - await client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName, String.Empty, "application/directory"); - } - else - { - var cloudHash = info.Hash.ToLower(); + var newFilePath = action.LocalFile.FullName; - var hash = action.LocalHash.Value; - var topHash = action.TopHash.Value; + //How do we handle concurrent renames and deletes/uploads/downloads? + //* A conflicting upload means that a file was renamed before it had a chance to finish uploading + // This should never happen as the network agent executes only one action at a time + //* A conflicting download means that the file was modified on the cloud. While we can go on and complete + // the rename, there may be a problem if the file is downloaded in blocks, as subsequent block requests for the + // same name will fail. + // This should never happen as the network agent executes only one action at a time. + //* A conflicting delete can happen if the rename was followed by a delete action that didn't have the chance + // to remove the rename from the queue. + // We can probably ignore this case. It will result in an error which should be ignored - //If the file hashes match, abort the upload - if (hash == cloudHash || topHash == cloudHash) - { - //but store any metadata changes - StatusKeeper.StoreInfo(fullFileName, info); - Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName); - return; - } - - - //Mark the file as modified while we upload it - StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified); - //And then upload it - //Upload even small files using the Hashmap. The server may already contain - //the relevant block + //The local file is already renamed + StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Modified).Wait(); - //First, calculate the tree hash - var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize, - accountInfo.BlockHash, 2); - - await UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash); - } - //If everything succeeds, change the file and overlay status to normal - StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal); - } - //Notify the Shell to update the overlays - NativeMethods.RaiseChangeNotification(fullFileName); - StatusNotification.NotifyChangedFile(fullFileName); - } - catch (AggregateException ex) - { - var exc = ex.InnerException as WebException; - if (exc == null) - throw ex.InnerException; - if (HandleUploadWebException(action, exc)) - return; - throw; - } - catch (WebException ex) - { - if (HandleUploadWebException(action, ex)) - return; - throw; - } - catch (Exception ex) - { - Log.Error("Unexpected error while uploading file", ex); - throw; - } - - } + var account = action.CloudFile.Account ?? accountInfo.UserName; + var container = action.CloudFile.Container; + var client = new CloudFilesClient(accountInfo); + //TODO: What code is returned when the source file doesn't exist? + client.MoveObject(account, container, action.OldCloudFile.Name, container, action.CloudFile.Name); - private bool HandleUploadWebException(CloudAction action, WebException exc) - { - var response = exc.Response as HttpWebResponse; - if (response == null) - throw exc; - if (response.StatusCode == HttpStatusCode.Unauthorized) - { - Log.Error("Not allowed to upload file", exc); - var message = String.Format("Not allowed to uplad file {0}", action.LocalFile.FullName); - StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Unchanged, FileOverlayStatus.Normal); - StatusNotification.NotifyChange(message, TraceLevel.Warning); - return true; + StatusKeeper.SetFileStatus(newFilePath, FileStatus.Unchanged); + StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Normal).Wait(); + NativeMethods.RaiseChangeNotification(newFilePath); } - return false; } - public async Task UploadWithHashMap(AccountInfo accountInfo,ObjectInfo cloudFile,FileInfo fileInfo,string url,TreeHash treeHash) - { - if (accountInfo == null) - throw new ArgumentNullException("accountInfo"); - if (cloudFile==null) - throw new ArgumentNullException("cloudFile"); - if (fileInfo == null) - throw new ArgumentNullException("fileInfo"); - if (String.IsNullOrWhiteSpace(url)) - throw new ArgumentNullException(url); - if (treeHash==null) - throw new ArgumentNullException("treeHash"); - if (String.IsNullOrWhiteSpace(cloudFile.Container) ) - throw new ArgumentException("Invalid container","cloudFile"); - Contract.EndContractBlock(); - - var fullFileName = fileInfo.GetProperCapitalization(); - - var account = cloudFile.Account ?? accountInfo.UserName; - var container = cloudFile.Container ; - - var client = new CloudFilesClient(accountInfo); - //Send the hashmap to the server - var missingHashes = await client.PutHashMap(account, container, url, treeHash); - //If the server returns no missing hashes, we are done - while (missingHashes.Count > 0) - { - - var buffer = new byte[accountInfo.BlockSize]; - foreach (var missingHash in missingHashes) - { - //Find the proper block - var blockIndex = treeHash.HashDictionary[missingHash]; - var offset = blockIndex*accountInfo.BlockSize; - - var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize); - - try - { - //And upload the block - await client.PostBlock(account, container, buffer, 0, read); - Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex, fullFileName); - } - catch (Exception exc) - { - Log.ErrorFormat("[ERROR] uploading block {0} of {1}\n{2}", blockIndex, fullFileName, exc); - } - - } - - //Repeat until there are no more missing hashes - missingHashes = await client.PutHashMap(account, container, url, treeHash); - } - } + - public void AddAccount(AccountInfo accountInfo) - { - if (!_accounts.Contains(accountInfo)) - _accounts.Add(accountInfo); - } }