}
// add some other attributes
- latestVersion.AppName = _config.ApplicationName;
- latestVersion.AppVersionInstalled = _config.InstalledVersion;
-
+ if (latestVersion != null)
+ {
+ latestVersion.AppName = _config.ApplicationName;
+ latestVersion.AppVersionInstalled = _config.InstalledVersion;
+ }
// go ahead
return latestVersion;
}
using System;
using System.Collections.Generic;
using System.ComponentModel.Composition;
+using System.IO;
using System.Linq;
using System.Text;
+using System.Windows;
using Caliburn.Micro;
using Pithos.Core;
using Pithos.Core.Agents;
public ShellViewModel Shell { get; set; }
[Import]
+ public IStatusKeeper StatusKeeper { get; set; }
+
+ [Import]
public NetworkAgent NetworkAgent { get; set; }
+ [Import]
+ public IStatusNotification StatusNotification { get; set; }
+
public void Resolve(IEnumerable<ConflictFile> conflicts)
{
KeepServer(conflicts.Where(c => c.Action == ConflictAction.KeepServer));
let info = monitor.GetObjectInfo(conflict.FilePath)
select new CloudDownloadAction(account, info, "Resolver");
- downloadActions.Apply(action=> StatusAgent.SetFileState(action.FileState.FilePath,FileStatus.Unchanged,FileOverlayStatus.Normal,"Resolve by downloading"));
+ foreach (var action in downloadActions)
+ {
+ try
+ {
+ action.LocalFile.Delete();
+ StatusAgent.SetFileState(action.FileState.FilePath, FileStatus.Unchanged, FileOverlayStatus.Normal, "Resolve by downloading");
+ }
+ catch (Exception exc)
+ {
+ MessageBox.Show(
+ String.Format("The file {0} is in use and can't be deleted", action.LocalFile.FullName),
+ "Pithos+. Can't delete file");
+ }
+
+ }
//downloadActions.Apply(NetworkAgent.Post);
}
let account = accounts.First(acc => conflict.FilePath.StartsWith(acc.AccountPath, StringComparison.InvariantCultureIgnoreCase))
let info = FileInfoExtensions.FromPath(conflict.FilePath)
select new CloudUploadAction(account, info, conflict.State,
- account.BlockSize, account.BlockHash,"Resolver",false);
-
- actions.Apply(action => StatusAgent.SetFileState(action.FileState.FilePath, FileStatus.Modified, FileOverlayStatus.Normal, "Resolve by downloading"));
+ account.BlockSize, account.BlockHash,"Resolver",false,new Progress<double>());
+
+ foreach (var action in actions)
+ {
+ DeleteCloudFile(action);
+ StatusAgent.SetFileState(action.FileState.FilePath, FileStatus.Modified, FileOverlayStatus.Normal, "Resolve by downloading");
+
+ }
+
+ //If C!=S, C!=L, S!=L, C!=Null, S!=Null and we delete the server file, we will cause an upload of the new file
//actions.Apply(NetworkAgent.Post);
}
+ private void DeleteCloudFile(CloudUploadAction action)
+ {
+ using (StatusNotification.GetNotifier("Deleting server {0}", "Deleted server {0}", Path.GetFileName(action.LocalFile.Name)))
+ {
+
+ StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Deleted,
+ FileOverlayStatus.Deleted, "");
+ NetworkAgent.DeleteAgent.DeleteCloudFile(action.AccountInfo, action.CloudFile);
+ StatusKeeper.ClearFileStatus(action.LocalFile.FullName);
+ }
+ }
//Keeping both versions means that we need to copy one of them
//somewhere and keep the other
private void KeepBoth(IEnumerable<ConflictFile> conflicts)
<Grid >
<Grid.ColumnDefinitions>
<ColumnDefinition />
+<!--
<ColumnDefinition Width="150"/>
+-->
</Grid.ColumnDefinitions>
<StackPanel Grid.Column="0" Margin="0,5">
<TextBlock x:Name="FilePath" Text="{Binding FilePath}" Margin="5,0"/>
<TextBlock FontStyle="Italic" Margin="5,0">Local Date:<TextBlock x:Name="LocalModified" FontStyle="Italic" Text="{Binding LocalModified}" Margin="5,0"/> </TextBlock>
</StackPanel>
</StackPanel>
+<!--
<ComboBox Grid.Column="1" x:Name="Action" Margin="5"
ItemsSource="{Binding Source={StaticResource ActionsList}}"
SelectedValue="{Binding Action}" VerticalAlignment="Top"
/>
+-->
</Grid>
</DataTemplate>
<StackPanel Orientation="Horizontal" Grid.Row="2" HorizontalAlignment="Right">
<Button Name="Refresh" Content="Refresh" Margin="5,5,80,5" Style="{StaticResource ButtonStyle}" IsDefault="False" />
<Button Name="Apply" Content="OK" Margin="5,5,10,5" Style="{StaticResource ButtonStyle}" IsDefault="False" />
+<!--
<Button Name="Cancel" Content="Cancel" Margin="5,5,10,5" Style="{StaticResource ButtonStyle}" IsCancel="True" />
+-->
</StackPanel>
</Grid>
public void Apply()
{
- var conflicts = from conflict in Conflicts
+ /* var conflicts = from conflict in Conflicts
where conflict.Action != ConflictAction.Defer
select conflict;
- Resolver.Resolve(conflicts);
+ Resolver.Resolve(conflicts);*/
TryClose();
}
<CheckBox Name="CurrentAccount_SelectiveSyncEnabled" Content="Selective Sync Enabled" Grid.Row="7" Grid.Column="1"/>
<StackPanel Orientation="Horizontal" Grid.Row="8" Grid.Column="1">
<Button Name="SelectiveSyncFolders" Width="100" Style="{StaticResource ButtonStyle}" Content="Selective Sync" />
- <Button Name="MoveAccountFolder" Content="Move ..." Width="100" Style="{StaticResource ButtonStyle}" Margin="20,5,5,5"/>
+ <Button Name="MoveAccountFolder" Content="Move ..." Width="100" Style="{StaticResource ButtonStyle}" Margin="20,5,5,5" Visibility="Hidden"/>
<Button Name="ClearAccountCache" Content="Clear Cache" Width="100" Style="{StaticResource ButtonStyle}"/>
</StackPanel>
</Grid>
return _pithosStatus;
}
- public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string shortHash = null)
+ public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string etag = null)
{
_overlayCache[path] = overlayStatus;
return Task.Factory.StartNew(()=>{});
_overlayCache.TryRemove(oldPath, out value);
}
- public void UpdateFileChecksum(string path, string shortHash, string checksum)
+ public void UpdateFileChecksum(string path, string etag, string checksum)
{
_checksums[path] = checksum;
}
{
throw new NotImplementedException();
}
+
+ public void UpdateLastMD5(FileInfo path, string etag)
+ {
+ throw new NotImplementedException();
+ }
}
}
client.DeleteObject(null, FolderConstants.PithosContainer, fileName);
- var treeHash = Signature.CalculateTreeHashAsync(filePath, accountInfo.BlockSize, accountInfo.BlockHash, 2);
+ var treeHash = Signature.CalculateTreeHashAsync(filePath, accountInfo.BlockSize, accountInfo.BlockHash, 2, new Progress<double>());
var cloudFile = new ObjectInfo {Account = account, Container = "pithos"};
var fileInfo = new FileInfo(filePath);
.Wait();
Assert.IsTrue(File.Exists(filePath));
- var treeHash = Signature.CalculateTreeHashAsync(filePath, accountInfo.BlockSize, accountInfo.BlockHash, 2);
+ var treeHash = Signature.CalculateTreeHashAsync(filePath, accountInfo.BlockSize, accountInfo.BlockHash, 2, new Progress<double>());
Assert.AreEqual(treeHash.TopHash, newHash.TopHash);
}
}
- public static string CalculateHash(this FileSystemInfo info,int blockSize,string algorithm)
+ public static string CalculateHash(this FileSystemInfo info,int blockSize,string algorithm,IProgress<double> progress )
{
if (info==null)
throw new ArgumentNullException("info");
if (!info.Exists)
return String.Empty;
- return Signature.CalculateTreeHash(info.FullName, blockSize, algorithm).TopHash.ToHashString();
+ return Signature.CalculateTreeHash(info.FullName, blockSize, algorithm,progress).TopHash.ToHashString();
}
Originator = originator;
}
- public CloudAction(AccountInfo accountInfo, CloudActionType action, FileSystemInfo localFile, ObjectInfo cloudFile, FileState state, int blockSize, string algorithm,object originator)
+ public CloudAction(AccountInfo accountInfo, CloudActionType action, FileSystemInfo localFile, ObjectInfo cloudFile, FileState state, int blockSize, string algorithm,object originator,IProgress<double> progress )
: this(accountInfo,action,originator)
{
if(blockSize<=0)
if (LocalFile == null)
return;
- TreeHash = new Lazy<TreeHash>(() => Signature.CalculateTreeHash(LocalFile, blockSize,algorithm),
+ TreeHash = new Lazy<TreeHash>(() => Signature.CalculateTreeHash(LocalFile, blockSize,algorithm,progress),
LazyThreadSafetyMode.ExecutionAndPublication);
}
public bool IsCreation { get; set; }
- public CloudUploadAction(AccountInfo accountInfo, FileSystemInfo fileInfo, FileState state, int blockSize, string algorithm,object originator,bool isCreation)
- : base(accountInfo, CloudActionType.UploadUnconditional,fileInfo,CreateObjectInfoFor(accountInfo,fileInfo),state,blockSize,algorithm,originator)
+ public CloudUploadAction(AccountInfo accountInfo, FileSystemInfo fileInfo, FileState state, int blockSize, string algorithm,object originator,bool isCreation,IProgress<double> progress )
+ : base(accountInfo, CloudActionType.UploadUnconditional,fileInfo,CreateObjectInfoFor(accountInfo,fileInfo),state,blockSize,algorithm,originator,progress)
{
IsCreation = isCreation;
}
\r
if (await WaitOrAbort(accountInfo,cloudFile, cancellationToken).ConfigureAwait(false))\r
return;\r
- \r
+\r
+ var fileName = Path.GetFileName(filePath);\r
+ var progress = new Progress<double>(d =>\r
+ StatusNotification.Notify(new StatusNotification(String.Format("Hashing for Download {0} of {1}", d, fileName))));\r
+\r
+\r
TreeHash localTreeHash;\r
- using (StatusNotification.GetNotifier("Hashing for Download {0}", "Hashed for Download {0}", Path.GetFileName(filePath)))\r
+ \r
+ using (StatusNotification.GetNotifier("Hashing for Download {0}", "Hashed for Download {0}", fileName))\r
{\r
\r
localTreeHash = Signature.CalculateTreeHashAsync(filePath,\r
accountInfo.BlockSize,\r
- accountInfo.BlockHash, 1);\r
+ accountInfo.BlockHash, 1,progress);\r
}\r
\r
var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath);\r
StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing, String.Format("Calculating hashmap for {0} before download", Path.GetFileName(localPath)));\r
//Calculate the file's treehash\r
\r
+ var fileName = Path.GetFileName(localPath);\r
+ var progress = new Progress<double>(d =>\r
+ StatusNotification.Notify(new StatusNotification(String.Format("Hashing for Download {0} of {1}", d, fileName))));\r
+\r
//TODO: Should pass cancellation token here\r
- var treeHash = localTreeHash ?? Signature.CalculateTreeHashAsync(localPath, (int)serverHash.BlockSize, serverHash.BlockHash, 2);\r
+ var treeHash = localTreeHash ?? Signature.CalculateTreeHashAsync(localPath, (int)serverHash.BlockSize, serverHash.BlockHash, 2,progress);\r
\r
//And compare it with the server's hash\r
var upHashes = serverHash.GetHashesAsStrings();\r
return true;\r
\r
var info = new FileInfo(localPath);\r
- var shortHash = info.ComputeShortHash(StatusNotification);\r
+ var etag = info.ComputeShortHash(StatusNotification);\r
//If the file is different from the stored state, we have a change\r
- if (localState.ShortHash != shortHash)\r
+ if (localState.ETag != etag)\r
return true;\r
//If the top hashes differ, we have a change\r
return (localState.Checksum != cloudFile.X_Object_Hash);\r
UpdateFileStatus(state);
UpdateOverlayStatus(state);
- UpdateFileChecksum(state);
+ UpdateLastMD5(state);
WorkflowAgent.Post(state);
}
catch (IOException exc)
switch (state.Status)
{
case FileStatus.Created:
- this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash).Wait();
+ this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ETag).Wait();
break;
case FileStatus.Modified:
- this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified, state.ShortHash).Wait();
+ this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified, state.ETag).Wait();
break;
case FileStatus.Deleted:
//this.StatusAgent.RemoveFileOverlayStatus(state.Path);
break;
case FileStatus.Renamed:
this.StatusKeeper.ClearFileStatus(state.OldPath);
- this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified, state.ShortHash).Wait();
+ this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified, state.ETag).Wait();
break;
case FileStatus.Unchanged:
- this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal, state.ShortHash).Wait();
+ this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal, state.ETag).Wait();
break;
}
using (StatusNotification.GetNotifier("Hashing {0}", "Finished Hashing {0}", info.Name))
{
- var shortHash = info.ComputeShortHash(StatusNotification);
+ var etag = info.ComputeShortHash(StatusNotification);
- string merkleHash = info.CalculateHash(StatusKeeper.BlockSize, StatusKeeper.BlockHash);
- StatusKeeper.UpdateFileChecksum(path, shortHash, merkleHash);
+ var progress = new Progress<double>(d =>
+ StatusNotification.Notify(new StatusNotification(String.Format("Hashing {0} of {1}", d, info.Name))));
+
+ string merkleHash = info.CalculateHash(StatusKeeper.BlockSize, StatusKeeper.BlockHash,progress);
+ StatusKeeper.UpdateFileChecksum(path, etag, merkleHash);
state.Hash = merkleHash;
return state;
\r
//Get the local files here \r
var agent = AgentLocator<FileAgent>.Get(accountInfo.AccountPath); \r
- var files = await LoadLocalFileTuples(accountInfo, accountBatch);\r
+ var files = LoadLocalFileTuples(accountInfo, accountBatch);\r
\r
var states = FileState.Queryable.ToList(); \r
\r
\r
//Process only the changes in the batch file, if one exists\r
var stateTuples = accountBatch==null?tuples:tuples.Where(t => accountBatch.Contains(t.FilePath));\r
- foreach (var tuple in stateTuples)\r
+ foreach (var tuple in stateTuples.Where(s=>!s.Locked))\r
{\r
await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
\r
return nextSince;\r
}\r
}\r
+/*\r
\r
private static void SetMerkleHash(AccountInfo accountInfo, StateTuple tuple)\r
{\r
//The Merkle hash for directories is that of an empty buffer\r
if (tuple.FileInfo is DirectoryInfo)\r
tuple.C = MERKLE_EMPTY;\r
- else if (tuple.FileState != null && tuple.MD5 == tuple.FileState.ShortHash)\r
+ else if (tuple.FileState != null && tuple.MD5 == tuple.FileState.ETag)\r
{\r
//If there is a state whose MD5 matches, load the merkle hash from the file state\r
//insteaf of calculating it\r
}\r
else\r
{\r
- tuple.Merkle = Signature.CalculateTreeHashAsync((FileInfo)tuple.FileInfo, accountInfo.BlockSize, accountInfo.BlockHash,1);\r
+ tuple.Merkle = Signature.CalculateTreeHashAsync((FileInfo)tuple.FileInfo, accountInfo.BlockSize, accountInfo.BlockHash,1,progress);\r
//tuple.C=tuple.Merkle.TopHash.ToHashString(); \r
}\r
}\r
+*/\r
\r
- private async Task<List<Tuple<FileSystemInfo, string>>> LoadLocalFileTuples(AccountInfo accountInfo,IEnumerable<string> batch )\r
+ private IEnumerable<FileSystemInfo> LoadLocalFileTuples(AccountInfo accountInfo,IEnumerable<string> batch )\r
{\r
using (ThreadContext.Stacks["Account Files Hashing"].Push(accountInfo.UserName))\r
{\r
.EnumerateFileSystemInfos();\r
if (batchPaths.Count>0)\r
localInfos= localInfos.Where(fi => batchPaths.Contains(fi.FullName));\r
- \r
- //Use the queue to retry locked file hashing\r
- var fileQueue = new ConcurrentQueue<FileSystemInfo>(localInfos);\r
- \r
\r
- var results = new List<Tuple<FileSystemInfo, string>>();\r
- var backoff = 0;\r
- while (fileQueue.Count > 0)\r
- {\r
- FileSystemInfo file;\r
- fileQueue.TryDequeue(out file);\r
- using (ThreadContext.Stacks["File"].Push(file.FullName))\r
- {\r
- try\r
- {\r
- //Replace MD5 here, do the calc while syncing individual files\r
- string hash ;\r
- if (file is DirectoryInfo)\r
- hash = MD5_EMPTY;\r
- else\r
- {\r
- //Wait in case the FileAgent has requested a Pause\r
- await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
- \r
- using (StatusNotification.GetNotifier("Hashing {0}", "", file.Name))\r
- {\r
- hash = ((FileInfo)file).ComputeShortHash(StatusNotification);\r
- backoff = 0;\r
- }\r
- } \r
- results.Add(Tuple.Create(file, hash));\r
- }\r
- catch (IOException exc)\r
- {\r
- Log.WarnFormat("[HASH] File in use, will retry [{0}]", exc);\r
- fileQueue.Enqueue(file);\r
- //If this is the only enqueued file \r
- if (fileQueue.Count != 1) continue;\r
- \r
- \r
- //Increase delay\r
- if (backoff<60000)\r
- backoff += 10000;\r
- //Pause Polling for the specified time\r
- }\r
- if (backoff>0)\r
- await PauseFor(backoff).ConfigureAwait(false);\r
- }\r
- }\r
-\r
- return results;\r
+ return localInfos;\r
}\r
}\r
\r
\r
private async Task SyncSingleItem(AccountInfo accountInfo, StateTuple tuple, FileAgent agent, CancellationToken token)\r
{\r
- Log.DebugFormat("Sync [{0}] C:[{1}] L:[{2}] S:[{3}]",tuple.FilePath,tuple.C,tuple.L,tuple.S);\r
+ Log.DebugFormat("Sync [{0}] C:[{1}] L:[{2}] S:[{3}]", tuple.FilePath, tuple.C, tuple.L, tuple.S);\r
+\r
+ try\r
+ {\r
\r
- var localFilePath = tuple.FilePath;\r
- //Don't use the tuple info, it may have been deleted\r
- var localInfo = FileInfoExtensions.FromPath(localFilePath);\r
+ var localFilePath = tuple.FilePath;\r
+ //Don't use the tuple info, it may have been deleted\r
+ var localInfo = FileInfoExtensions.FromPath(localFilePath);\r
\r
\r
- var isUnselectedRootFolder = agent.IsUnselectedRootFolder(tuple.FilePath);\r
+ var isUnselectedRootFolder = agent.IsUnselectedRootFolder(tuple.FilePath);\r
\r
- //Unselected root folders that have not yet been uploaded should be uploaded and added to the \r
- //selective folders\r
+ //Unselected root folders that have not yet been uploaded should be uploaded and added to the \r
+ //selective folders\r
\r
- if (!Selectives.IsSelected(accountInfo, localFilePath) && !(isUnselectedRootFolder && tuple.ObjectInfo==null) ) \r
- return;\r
+ if (!Selectives.IsSelected(accountInfo, localFilePath) &&\r
+ !(isUnselectedRootFolder && tuple.ObjectInfo == null))\r
+ return;\r
\r
- // Local file unchanged? If both C and L are null, make sure it's because \r
- //both the file is missing and the state checksum is not missing\r
- if (tuple.C == tuple.L /*&& (localInfo.Exists || tuple.FileState == null)*/)\r
- {\r
- //No local changes\r
- //Server unchanged?\r
- if (tuple.S == tuple.L)\r
- {\r
- // No server changes\r
- //Has the file been renamed on the server?\r
- MoveForServerMove(accountInfo, tuple);\r
- }\r
- else\r
+ // Local file unchanged? If both C and L are null, make sure it's because \r
+ //both the file is missing and the state checksum is not missing\r
+ if (tuple.C == tuple.L /*&& (localInfo.Exists || tuple.FileState == null)*/)\r
{\r
- //Different from server\r
- //Does the server file exist?\r
- if (tuple.S == null)\r
+ //No local changes\r
+ //Server unchanged?\r
+ if (tuple.S == tuple.L)\r
{\r
- //Server file doesn't exist\r
- //deleteObjectFromLocal()\r
- using (StatusNotification.GetNotifier("Deleting local {0}", "Deleted local {0}", Path.GetFileName(localFilePath)))\r
+ // No server changes\r
+ //Has the file been renamed on the server?\r
+ MoveForServerMove(accountInfo, tuple);\r
+ }\r
+ else\r
+ {\r
+ //Different from server\r
+ //Does the server file exist?\r
+ if (tuple.S == null)\r
{\r
- StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
- FileOverlayStatus.Deleted, "");\r
- using (NetworkGate.Acquire(localFilePath, NetworkOperation.Deleting))\r
+ //Server file doesn't exist\r
+ //deleteObjectFromLocal()\r
+ using (\r
+ StatusNotification.GetNotifier("Deleting local {0}", "Deleted local {0}",\r
+ Path.GetFileName(localFilePath)))\r
{\r
- agent.Delete(localFilePath);\r
+ StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
+ FileOverlayStatus.Deleted, "");\r
+ using (NetworkGate.Acquire(localFilePath, NetworkOperation.Deleting))\r
+ {\r
+ agent.Delete(localFilePath);\r
+ }\r
+ //updateRecord(Remove C, L)\r
+ StatusKeeper.ClearFileStatus(localFilePath);\r
}\r
- //updateRecord(Remove C, L)\r
- StatusKeeper.ClearFileStatus(localFilePath);\r
}\r
- }\r
- else\r
- {\r
- //Server file exists\r
- //downloadServerObject() // Result: L = S\r
- //If the file has moved on the server, move it locally before downloading\r
- using (StatusNotification.GetNotifier("Downloading {0}", "Downloaded {0}", Path.GetFileName(localFilePath)))\r
+ else\r
{\r
- var targetPath = MoveForServerMove(accountInfo, tuple);\r
+ //Server file exists\r
+ //downloadServerObject() // Result: L = S\r
+ //If the file has moved on the server, move it locally before downloading\r
+ using (\r
+ StatusNotification.GetNotifier("Downloading {0}", "Downloaded {0}",\r
+ Path.GetFileName(localFilePath)))\r
+ {\r
+ var targetPath = MoveForServerMove(accountInfo, tuple);\r
\r
- StatusKeeper.SetFileState(targetPath, FileStatus.Modified,FileOverlayStatus.Modified, "");\r
- \r
- await NetworkAgent.Downloader.DownloadCloudFile(accountInfo, tuple.ObjectInfo, targetPath, token)\r
- .ConfigureAwait(false);\r
- //updateRecord( L = S )\r
- StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag,\r
- tuple.ObjectInfo.X_Object_Hash);\r
+ StatusKeeper.SetFileState(targetPath, FileStatus.Modified, FileOverlayStatus.Modified,\r
+ "");\r
\r
- StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
- }\r
+ await\r
+ NetworkAgent.Downloader.DownloadCloudFile(accountInfo, tuple.ObjectInfo, targetPath,\r
+ token)\r
+ .ConfigureAwait(false);\r
+ //updateRecord( L = S )\r
+ StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag,\r
+ tuple.ObjectInfo.X_Object_Hash);\r
\r
- /*\r
- StatusKeeper.SetFileState(targetPath, FileStatus.Unchanged,\r
- FileOverlayStatus.Normal, "");\r
- */\r
- }\r
- }\r
+ StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
+ }\r
\r
- }\r
- else\r
- {\r
- //Local changes found\r
+ /*\r
+ StatusKeeper.SetFileState(targetPath, FileStatus.Unchanged,\r
+ FileOverlayStatus.Normal, "");\r
+ */\r
+ }\r
+ }\r
\r
- //Server unchanged?\r
- if (tuple.S == tuple.L)\r
+ }\r
+ else\r
{\r
- //The FileAgent selective sync checks for new root folder files\r
- if (!agent.Ignore(localFilePath))\r
+ //Local changes found\r
+\r
+ //Server unchanged?\r
+ if (tuple.S == tuple.L)\r
{\r
- if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
- {\r
- //deleteObjectFromServer()\r
- DeleteCloudFile(accountInfo, tuple);\r
- //updateRecord( Remove L, S) \r
- }\r
- else\r
+ //The FileAgent selective sync checks for new root folder files\r
+ if (!agent.Ignore(localFilePath))\r
{\r
- //uploadLocalObject() // Result: S = C, L = S \r
-\r
- //Debug.Assert(tuple.FileState !=null);\r
- var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
- accountInfo.BlockSize, accountInfo.BlockHash,\r
- "Poll", isUnselectedRootFolder);\r
- using (StatusNotification.GetNotifier("Uploading {0}", "Uploaded {0}", Path.GetFileName(localFilePath)))\r
+ if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
{\r
- await NetworkAgent.Uploader.UploadCloudFile(action, token).ConfigureAwait(false);\r
+ //deleteObjectFromServer()\r
+ DeleteCloudFile(accountInfo, tuple);\r
+ //updateRecord( Remove L, S) \r
}\r
+ else\r
+ {\r
+ //uploadLocalObject() // Result: S = C, L = S \r
+ var progress = new Progress<double>(d =>\r
+ StatusNotification.Notify(new StatusNotification(String.Format("Merkle Hashing for Upload {0:p} of {1}", d, localInfo.Name))));\r
+\r
+ //Debug.Assert(tuple.FileState !=null);\r
+ var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
+ accountInfo.BlockSize, accountInfo.BlockHash,\r
+ "Poll", isUnselectedRootFolder,progress);\r
+ using (\r
+ StatusNotification.GetNotifier("Uploading {0}", "Uploaded {0}",\r
+ Path.GetFileName(localFilePath)))\r
+ {\r
+ await NetworkAgent.Uploader.UploadCloudFile(action, token).ConfigureAwait(false);\r
+ }\r
\r
- //updateRecord( S = C )\r
- //State updated by the uploader\r
+ //updateRecord( S = C )\r
+ //State updated by the uploader\r
\r
- if (isUnselectedRootFolder)\r
- {\r
- ProcessChildren(accountInfo, tuple, agent, token);\r
+ if (isUnselectedRootFolder)\r
+ {\r
+ ProcessChildren(accountInfo, tuple, agent, token);\r
+ }\r
}\r
}\r
}\r
- }\r
- else\r
- {\r
- if (tuple.C == tuple.S)\r
- {\r
- // (Identical Changes) Result: L = S\r
- //doNothing()\r
- //Detect server moves\r
- var targetPath = MoveForServerMove(accountInfo, tuple);\r
- StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
- }\r
else\r
{\r
- if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
- {\r
- //deleteObjectFromServer()\r
- DeleteCloudFile(accountInfo, tuple);\r
- //updateRecord(Remove L, S) \r
- }\r
- //If both the local and server files are missing, the state is stale\r
- else if (!localInfo.Exists && (tuple.S == null || tuple.ObjectInfo == null))\r
+ if (tuple.C == tuple.S)\r
{\r
- StatusKeeper.ClearFileStatus(localInfo.FullName);\r
+ // (Identical Changes) Result: L = S\r
+ //doNothing()\r
+ //Detect server moves\r
+ var targetPath = MoveForServerMove(accountInfo, tuple);\r
+ StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
}\r
else\r
{\r
- ReportConflictForMismatch(localFilePath);\r
- //identifyAsConflict() // Manual action required\r
+ if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
+ {\r
+ //deleteObjectFromServer()\r
+ DeleteCloudFile(accountInfo, tuple);\r
+ //updateRecord(Remove L, S) \r
+ }\r
+ //If both the local and server files are missing, the state is stale\r
+ else if (!localInfo.Exists && (tuple.S == null || tuple.ObjectInfo == null))\r
+ {\r
+ StatusKeeper.ClearFileStatus(localInfo.FullName);\r
+ }\r
+ else\r
+ {\r
+ ReportConflictForMismatch(localFilePath);\r
+ //identifyAsConflict() // Manual action required\r
+ }\r
}\r
}\r
}\r
}\r
+ catch (Exception exc)\r
+ {\r
+ //In case of error log and retry with the next poll\r
+ Log.ErrorFormat("[SYNC] Failed for file {0}. Will Retry.\r\n{1}",tuple.FilePath,exc);\r
+\r
+ \r
+ }\r
}\r
\r
private string MoveForServerMove(AccountInfo accountInfo, StateTuple tuple)\r
fileTuples.ApplyAction(async t => await SyncSingleItem(accountInfo, t, agent, token).ConfigureAwait(false));\r
}\r
\r
- private static IEnumerable<StateTuple> MergeSources(\r
+\r
+ /*\r
+ * //Use the queue to retry locked file hashing\r
+ var fileQueue = new ConcurrentQueue<FileSystemInfo>(localInfos);\r
+ \r
+\r
+ var results = new List<Tuple<FileSystemInfo, string>>();\r
+ var backoff = 0;\r
+ while (fileQueue.Count > 0)\r
+ {\r
+ FileSystemInfo file;\r
+ fileQueue.TryDequeue(out file);\r
+ using (ThreadContext.Stacks["File"].Push(file.FullName))\r
+ {\r
+ try\r
+ {\r
+ //Replace MD5 here, do the calc while syncing individual files\r
+ string hash ;\r
+ if (file is DirectoryInfo)\r
+ hash = MD5_EMPTY;\r
+ else\r
+ {\r
+ //Wait in case the FileAgent has requested a Pause\r
+ await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
+ \r
+ using (StatusNotification.GetNotifier("Hashing {0}", "", file.Name))\r
+ {\r
+ hash = ((FileInfo)file).ComputeShortHash(StatusNotification);\r
+ backoff = 0;\r
+ }\r
+ } \r
+ results.Add(Tuple.Create(file, hash));\r
+ }\r
+ catch (IOException exc)\r
+ {\r
+ Log.WarnFormat("[HASH] File in use, will retry [{0}]", exc);\r
+ fileQueue.Enqueue(file);\r
+ //If this is the only enqueued file \r
+ if (fileQueue.Count != 1) continue;\r
+ \r
+ \r
+ //Increase delay\r
+ if (backoff<60000)\r
+ backoff += 10000;\r
+ //Pause Polling for the specified time\r
+ }\r
+ if (backoff>0)\r
+ await PauseFor(backoff).ConfigureAwait(false);\r
+ }\r
+ }\r
+\r
+ return results;\r
+\r
+ */\r
+ private IEnumerable<StateTuple> MergeSources(\r
IEnumerable<Tuple<string, ObjectInfo>> infos, \r
- IEnumerable<Tuple<FileSystemInfo, string>> files, \r
+ IEnumerable<FileSystemInfo> files, \r
IEnumerable<FileState> states)\r
{\r
- var tuplesByPath = new Dictionary<string, StateTuple>();\r
- foreach (var file in files)\r
- {\r
- var fsInfo = file.Item1;\r
- var fileHash = fsInfo is DirectoryInfo? MD5_EMPTY:file.Item2;\r
+ var tuplesByPath = files.ToDictionary(f => f.FullName, f => new StateTuple {FileInfo = f}); new Dictionary<string, StateTuple>();\r
\r
- tuplesByPath[fsInfo.FullName] = new StateTuple {FileInfo = fsInfo, C=fileHash,MD5 = fileHash};\r
- }\r
+ //For files that have state\r
foreach (var state in states)\r
{\r
StateTuple hashTuple;\r
if (tuplesByPath.TryGetValue(state.FilePath, out hashTuple))\r
{\r
hashTuple.FileState = state;\r
+ UpdateMD5(hashTuple);\r
}\r
else\r
{\r
tuplesByPath[state.FilePath] = hashTuple;\r
}\r
}\r
+ //for files that don't have state\r
+ foreach (var tuple in tuplesByPath.Values.Where(t => t.FileState == null))\r
+ {\r
+ UpdateMD5(tuple);\r
+ }\r
\r
var tuplesByID = tuplesByPath.Values\r
.Where(tuple => tuple.FileState != null && tuple.FileState.ObjectID!=null)\r
return tuplesByPath.Values;\r
}\r
\r
+ private void UpdateMD5(StateTuple hashTuple)\r
+ {\r
+ \r
+ try\r
+ {\r
+ var hash = Signature.MD5_EMPTY;\r
+ if (hashTuple.FileInfo is FileInfo)\r
+ {\r
+ var file = hashTuple.FileInfo as FileInfo;\r
+ var stateDate = hashTuple.NullSafe(h => h.FileState).NullSafe(s => s.LastWriteDate) ??\r
+ DateTime.MinValue;\r
+ if (file.LastWriteTime - stateDate < TimeSpan.FromSeconds(1) &&\r
+ hashTuple.FileState.LastLength == file.Length)\r
+ {\r
+ hash = hashTuple.FileState.LastMD5;\r
+ }\r
+ else\r
+ {\r
+ //Modified, must calculate hash\r
+ hash = file.ComputeShortHash(StatusNotification);\r
+ StatusKeeper.UpdateLastMD5(file, hash);\r
+ }\r
+ }\r
+ hashTuple.C = hash;\r
+ hashTuple.MD5 = hash;\r
+ }\r
+ catch (IOException)\r
+ {\r
+ hashTuple.Locked = true;\r
+ } \r
+ }\r
+\r
/// <summary>\r
/// Returns the latest LastModified date from the list of objects, but only if it is before\r
/// than the threshold value\r
private bool _pause;\r
\r
const string MERKLE_EMPTY = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";\r
- const string MD5_EMPTY = "d41d8cd98f00b204e9800998ecf8427e";\r
\r
\r
private void ReportConflictForMismatch(string localFilePath)\r
{
get
{
- var hash = FileState.NullSafe(f => f.ShortHash);
+ var hash = FileState.NullSafe(f => f.ETag);
return String.IsNullOrWhiteSpace(hash) ? null : hash;
}
}
}
}
+ public bool Locked { get; set; }
+
public StateTuple() { }
public StateTuple(FileSystemInfo info)
//dictionary
//If the hashes don't match the file was changed
- if (fileState.ShortHash != hashString)
+ if (fileState.ETag != hashString)
{
_persistenceAgent.Post(() => UpdateStatusDirect((Guid) fileState.Id, FileStatus.Modified));
}
{
using (var connection = GetConnection())
- using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,ShortHash,Version ,VersionTimeStamp,IsShared ,SharedBy ,ShareWrite from FileState where FilePath=:path COLLATE NOCASE", connection))
+ using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,ETag,Version ,VersionTimeStamp,IsShared ,SharedBy ,ShareWrite from FileState where FilePath=:path COLLATE NOCASE", connection))
{
command.Parameters.AddWithValue("path", path);
OverlayStatus =reader.IsDBNull(2)?FileOverlayStatus.Unversioned: (FileOverlayStatus) reader.GetInt64(2),
FileStatus = reader.IsDBNull(3)?FileStatus.Missing:(FileStatus) reader.GetInt64(3),
Checksum = reader.IsDBNull(4)?"":reader.GetString(4),
- ShortHash= reader.IsDBNull(5)?"":reader.GetString(5),
+ ETag= reader.IsDBNull(5)?"":reader.GetString(5),
Version = reader.IsDBNull(6)?default(long):reader.GetInt64(6),
VersionTimeStamp = reader.IsDBNull(7)?default(DateTime):reader.GetDateTime(7),
IsShared = !reader.IsDBNull(8) && reader.GetBoolean(8),
_persistenceAgent.Post(() => FileState.StoreOverlayStatus(path,overlayStatus));
}*/
- public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string shortHash = null)
+ public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string etag = null)
{
if (String.IsNullOrWhiteSpace(path))
throw new ArgumentNullException("path");
throw new ArgumentException("The path must be rooted","path");
Contract.EndContractBlock();
- return _persistenceAgent.PostAndAwait(() => FileState.StoreOverlayStatus(path,overlayStatus,shortHash));
+ return _persistenceAgent.PostAndAwait(() => FileState.StoreOverlayStatus(path,overlayStatus,etag));
}
/* public void RenameFileOverlayStatus(string oldPath, string newPath)
//If the ID exists, update the status
if (StateExistsByID(objectInfo.UUID,connection))
command.CommandText =
- "update FileState set FilePath=:path,FileStatus= :fileStatus, Checksum=:checksum, ShortHash=:shortHash,Version=:version,VersionTimeStamp=:versionTimeStamp where ObjectID = :objectID ";
+ "update FileState set FilePath=:path,FileStatus= :fileStatus, Checksum=:checksum, ETag=:etag,LastMD5=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp where ObjectID = :objectID ";
else if (StateExists(path, connection))
//If the ID doesn't exist, try to update using the path, and store the ID as well.
command.CommandText =
- "update FileState set FileStatus= :fileStatus, ObjectID=:objectID, Checksum=:checksum, ShortHash=:shortHash,Version=:version,VersionTimeStamp=:versionTimeStamp where FilePath = :path COLLATE NOCASE ";
+ "update FileState set FileStatus= :fileStatus, ObjectID=:objectID, Checksum=:checksum, ETag=:etag,LastMD5=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp where FilePath = :path COLLATE NOCASE ";
else
{
command.CommandText =
- "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,ShortHash,FileStatus,OverlayStatus,ObjectID) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:shortHash,:fileStatus,:overlayStatus,:objectID)";
+ "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,ETag,LastMD5,FileStatus,OverlayStatus,ObjectID) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:etag,:etag,:fileStatus,:overlayStatus,:objectID)";
command.Parameters.AddWithValue("id", Guid.NewGuid());
}
command.Parameters.AddWithValue("path", path);
command.Parameters.AddWithValue("checksum", objectInfo.X_Object_Hash);
- command.Parameters.AddWithValue("shortHash", objectInfo.ETag);
+ command.Parameters.AddWithValue("etag", objectInfo.ETag);
command.Parameters.AddWithValue("version", objectInfo.Version);
command.Parameters.AddWithValue("versionTimeStamp", objectInfo.VersionTimestamp);
command.Parameters.AddWithValue("fileStatus", FileStatus.Unchanged);
}
}
- public void UpdateFileChecksum(string path, string shortHash, string checksum)
+ public void UpdateFileChecksum(string path, string etag, string checksum)
{
if (String.IsNullOrWhiteSpace(path))
throw new ArgumentNullException("path");
throw new ArgumentException("The path must be rooted", "path");
Contract.EndContractBlock();
- _persistenceAgent.Post(() => FileState.UpdateChecksum(path, shortHash,checksum));
+ _persistenceAgent.Post(() => FileState.UpdateChecksum(path, etag,checksum));
+ }
+
+ public void UpdateLastMD5(FileInfo file, string etag)
+ {
+ if (file==null)
+ throw new ArgumentNullException("file");
+ if (String.IsNullOrWhiteSpace(etag))
+ throw new ArgumentNullException("etag");
+ Contract.EndContractBlock();
+
+ _persistenceAgent.Post(() => FileState.UpdateLastMD5(file, etag));
}
\r
var fileInfo = action.LocalFile;\r
\r
+ var progress=new Progress<double>(d=>\r
+ StatusNotification.Notify(new StatusNotification(String.Format("Merkle Hashing for Upload {0:p} of {1}",d,fileInfo.Name))));\r
\r
TreeHash localTreeHash;\r
- using (StatusNotification.GetNotifier("Hashing for Upload {0}", "Hashed for Upload {0}", fileInfo.Name))\r
+ using (StatusNotification.GetNotifier("Merkle Hashing for Upload {0}", "Merkle Hashed for Upload {0}", fileInfo.Name))\r
{\r
localTreeHash = Signature.CalculateTreeHashAsync(fileInfo.FullName,\r
action.AccountInfo.BlockSize,\r
- action.AccountInfo.BlockHash, 1);\r
+ action.AccountInfo.BlockHash, 1,progress);\r
}\r
\r
if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))\r
/// </summary>
/// <remarks>The algorithm used to calculate this hash should be cheap</remarks>
[Property(NotNull = true, Default = "")]
- public string ShortHash { get; set; }
+ public string ETag { get; set; }
+
+ [Property(NotNull = true, Default = "")]
+ public string LastMD5 { get; set; }
[Property]
- public DateTime? ShortHashDate { get; set; }
-
+ public DateTime? LastWriteDate { get; set; }
+
+ [Property]
+ public long? LastLength { get; set; }
[Property]
public long? Version { get; set; }
FilePath = absolutePath,
Id = Guid.NewGuid(),
OverlayStatus = newStatus,
- ShortHash = String.Empty,
+ ETag = String.Empty,
IsFolder=Directory.Exists(absolutePath)
};
newState.CreateAndFlush();
}
*/
- public static void StoreOverlayStatus(string absolutePath, FileOverlayStatus newStatus,string shortHash)
+ public static void StoreOverlayStatus(string absolutePath, FileOverlayStatus newStatus,string etag)
{
if (string.IsNullOrWhiteSpace(absolutePath))
throw new ArgumentNullException("absolutePath");
FilePath = absolutePath,
Id = Guid.NewGuid(),
OverlayStatus = newStatus,
- ShortHash = shortHash??String.Empty,
+ ETag = etag??String.Empty,
IsFolder=Directory.Exists(absolutePath)
};
newState.CreateAndFlush();
}, null);
}*/
- public static void UpdateChecksum(string absolutePath, string shortHash, string checksum)
+ public static void UpdateChecksum(string absolutePath, string etag, string checksum)
{
if (string.IsNullOrWhiteSpace(absolutePath))
throw new ArgumentNullException("absolutePath");
ExecuteWithRetry((session, instance) =>
{
- const string hqlUpdate = "update FileState set Checksum= :checksum,ShortHash=:shortHash where FilePath = :path ";
+ const string hqlUpdate = "update FileState set Checksum= :checksum,ETag=:etag where FilePath = :path ";
var updatedEntities = session.CreateQuery(hqlUpdate)
.SetString("path", absolutePath)
.SetString("checksum", checksum)
- .SetString("shortHash", shortHash)
+ .SetString("etag", etag)
.ExecuteUpdate();
return updatedEntities;
}, null);
}
+ public static void UpdateLastMD5(FileInfo file, string md5)
+ {
+ if (file==null)
+ throw new ArgumentNullException("file");
+ Contract.EndContractBlock();
+
+ ExecuteWithRetry((session, instance) =>
+ {
+ const string hqlUpdate = "update FileState set LastMD5=:md5, LastWriteDate=:date,LastLength=:length where FilePath = :path ";
+ var fullName = file.WithProperCapitalization().FullName;
+
+ var updatedEntities = session.CreateQuery(hqlUpdate)
+ .SetDateTime("date", file.LastWriteTime)
+ .SetInt64("length", file.Length)
+ .SetString("md5", md5)
+ .SetString("path", fullName)
+ .ExecuteUpdate();
+ if (updatedEntities == 0)
+ {
+ var newState = new FileState
+ {
+ FilePath = fullName,
+ Id = Guid.NewGuid(),
+ OverlayStatus = FileOverlayStatus.Normal,
+ FileStatus=FileStatus.Unchanged,
+ IsFolder = false,
+ LastLength=file.Length,
+ LastWriteDate=file.LastWriteTime,
+ LastMD5 = md5 ?? String.Empty,
+ ETag=String.Empty
+ };
+ newState.CreateAndFlush();
+ }
+ return updatedEntities;
+ }, null);
+
+ }
+
public static void ChangeRootPath(string oldPath, string newPath)
{
if (String.IsNullOrWhiteSpace(oldPath))
FilePath = info.FullName,
OverlayStatus = FileOverlayStatus.Unversioned,
FileStatus = FileStatus.Created,
- ShortHash=String.Empty,
+ ETag=String.Empty,
Id = Guid.NewGuid()
};
- var shortHash = ((FileInfo)info).ComputeShortHash(notification);
+ var etag = ((FileInfo)info).ComputeShortHash(notification);
var fileState = new FileState
{
FilePath = info.FullName,
OverlayStatus = FileOverlayStatus.Unversioned,
FileStatus = FileStatus.Created,
- ShortHash=shortHash,
+ ETag=etag,
Id = Guid.NewGuid()
};
return fileState;
[ContractClass(typeof(IStatusKeeperContract))]
public interface IStatusKeeper
{
- Task SetFileOverlayStatus(string path, FileOverlayStatus status, string shortHash = null);
- void UpdateFileChecksum(string path, string shortHash, string checksum);
+ Task SetFileOverlayStatus(string path, FileOverlayStatus status, string etag = null);
+ void UpdateFileChecksum(string path, string etag, string checksum);
void SetFileStatus(string path, FileStatus status);
FileStatus GetFileStatus(string path);
void ClearFileStatus(string path);
void CleanupStaleStates(Network.AccountInfo accountInfo, List<ObjectInfo> objectInfos);
void CleanupOrphanStates();
+ void UpdateLastMD5(FileInfo path, string etag);
}
[ContractClassFor(typeof(IStatusKeeper))]
public abstract class IStatusKeeperContract : IStatusKeeper
{
- public Task SetFileOverlayStatus(string path, FileOverlayStatus status, string shortHash = null)
+ public Task SetFileOverlayStatus(string path, FileOverlayStatus status, string etag = null)
{
Contract.Requires(!String.IsNullOrWhiteSpace(path));
Contract.Requires(Path.IsPathRooted(path));
return default(Task);
}
- public void UpdateFileChecksum(string path, string shortHash, string checksum)
+ public void UpdateFileChecksum(string path, string etag, string checksum)
{
Contract.Requires(!String.IsNullOrWhiteSpace(path));
Contract.Requires(checksum!=null);
public void CleanupOrphanStates()
{
}
+
+ public void UpdateLastMD5(FileInfo path, string etag)
+ {
+
+ }
}
}
public string Hash { get; set; }
public string LastUpdateHash { get; set; }
- public string ShortHash { get; set; }
+ public string ETag { get; set; }
/*
public WorkflowState(AccountInfo accountInfo)
Path = state.FilePath.ToLower();
FileName = System.IO.Path.GetFileName(state.FilePath).ToLower();
Hash = state.Checksum;
- ShortHash = state.ShortHash;
+ ETag = state.ETag;
Status = state.OverlayStatus == FileOverlayStatus.Unversioned
? FileStatus.Created
: state.FileStatus;
};
client.Authenticate();
var fileName = @"vlc-1.1.11-win32.exe";
- var treeHash=Signature.CalculateTreeHashAsync(Path.Combine(@"e:\pithos\" ,fileName), 4*1024*1024 , "sha256", 2);
+ var treeHash = Signature.CalculateTreeHashAsync(Path.Combine(@"e:\pithos\", fileName), 4 * 1024 * 1024, "sha256", 2, new Progress<double>());
var result = client.PutHashMap(account, "pithos", fileName, treeHash).Result;
Assert.AreEqual(0,result.Count);
var fileSize = new FileInfo(file).Length;
var numBlocks = decimal.Ceiling(fileSize/blockSize);
- var md5 = Signature.CalculateMD5(file);
+ var md5 = Signature.CalculateMD5(file);
- var hash1 = Signature.CalculateTreeHashAsync(file, (int) blockSize,"sha256", 2);
+ var hash1 = Signature.CalculateTreeHashAsync(file, (int)blockSize, "sha256", 2, new Progress<double>());
Assert.IsNotNull(hash1.Hashes);
Assert.AreEqual(numBlocks, hash1.Hashes.Count());
var md5 = Signature.CalculateMD5(file);
- var hash1 = Signature.CalculateTreeHashAsync(file, (int) blockSize, "sha256", 2);
+ var hash1 = Signature.CalculateTreeHashAsync(file, (int)blockSize, "sha256", 2, new Progress<double>());
hash1.FileId = Guid.NewGuid();
var task = hash1.Save(@"e:\")
.ContinueWith(_ => TreeHash.LoadTreeHash(@"e:\", hash1.FileId)).Unwrap();
decimal blockSize = 4 * 1048576;
Trace.WriteLine("1");
- var stopwatch = Stopwatch.StartNew();
- var hash1 = Signature.CalculateTreeHashAsync(file, (int)blockSize, "sha256", 1);
+ var stopwatch = Stopwatch.StartNew();
+ var hash1 = Signature.CalculateTreeHashAsync(file, (int)blockSize, "sha256", 1, new Progress<double>());
stopwatch.Stop();
Trace.WriteLine(stopwatch.Elapsed);
Trace.WriteLine("2");
- stopwatch.Restart();
- var hash2 = Signature.CalculateTreeHashAsync(file, (int)blockSize, "sha256", 2);
+ stopwatch.Restart();
+ var hash2 = Signature.CalculateTreeHashAsync(file, (int)blockSize, "sha256", 2, new Progress<double>());
stopwatch.Stop();
Trace.WriteLine(stopwatch.Elapsed);
Trace.WriteLine("3");
stopwatch.Restart();
- var hash3 = Signature.CalculateTreeHashAsync(file, (int)blockSize, "sha256", 3);
+ var hash3 = Signature.CalculateTreeHashAsync(file, (int)blockSize, "sha256", 3, new Progress<double>());
stopwatch.Stop();
Trace.WriteLine(stopwatch.Elapsed);
client.UsePithos = true;
client.Authenticate();
var fileName = @"vlc-1.1.11-win32.exe";
- var localHash= Signature.CalculateTreeHashAsync(Path.Combine(@"e:\pithos\", fileName), 4 * 1024 * 1024, "sha256", 2);
+ var localHash= Signature.CalculateTreeHashAsync(Path.Combine(@"e:\pithos\", fileName), 4 * 1024 * 1024, "sha256", 2,new Progress<double>());
var upHash= client.GetHashMap(fileName, account, "pithos").Result;
Assert.AreEqual(upHash.TopHash, localHash.TopHash);
{
var fileName = @"vlc-1.1.11-win32.exe";
- var syncHash= Signature.CalculateTreeHash(Path.Combine(@"e:\pithos\", fileName), 4 * 1024 * 1024, "sha256");
- var asyncHash = Signature.CalculateTreeHashAsync(Path.Combine(@"e:\pithos\", fileName), 4 * 1024 * 1024, "sha256", 2)
+ var syncHash= Signature.CalculateTreeHash(Path.Combine(@"e:\pithos\", fileName), 4 * 1024 * 1024, "sha256",new Progress<double>());
+ var asyncHash = Signature.CalculateTreeHashAsync(Path.Combine(@"e:\pithos\", fileName), 4 * 1024 * 1024, "sha256", 2, new Progress<double>())
;
Assert.AreEqual(syncHash.TopHash, asyncHash.TopHash);
return _bufferMgr;\r
}\r
\r
- public static async Task<ConcurrentDictionary<long, byte[]>> CalculateBlockHashesInPlacePFor(FileStream stream, int blockSize, string algorithm, int parallelism)\r
+ public static async Task<ConcurrentDictionary<long, byte[]>> CalculateBlockHashesInPlacePFor(FileStream stream, int blockSize, string algorithm, int parallelism,IProgress<double> progress )\r
{\r
if (stream == null)\r
throw new ArgumentNullException("stream");\r
(double)filePosition / size);\r
*/\r
hashes[filePosition] = hash;\r
+ progress.Report((long)hashes.Count*blockSize*1.0/stream.Length);\r
});\r
}\r
bufIdx = (bufIdx + 1)%parallelism;\r
{
private static readonly log4net.ILog Log = log4net.LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
- public static string CalculateMD5(FileInfo info)
+ public const string MD5_EMPTY = "d41d8cd98f00b204e9800998ecf8427e";
+
+
+ public static string CalculateMD5(FileSystemInfo info)
{
if (info==null)
throw new ArgumentNullException("info");
throw new ArgumentException("info.FullName is empty","info");
Contract.EndContractBlock();
+ if (info is DirectoryInfo)
+ return MD5_EMPTY;
+
return CalculateMD5(info.FullName);
}
return shb.ToString().ToLower();
}
- public static TreeHash CalculateTreeHash(FileSystemInfo fileInfo, int blockSize, string algorithm)
+ public static TreeHash CalculateTreeHash(FileSystemInfo fileInfo, int blockSize, string algorithm,IProgress<double> progress )
{
if (fileInfo == null)
throw new ArgumentNullException("fileInfo");
if (fileInfo is DirectoryInfo || !fileInfo.Exists)
return TreeHash.Empty;
- return CalculateTreeHash(fileInfo.FullName, blockSize, algorithm);
+ return CalculateTreeHash(fileInfo.FullName, blockSize, algorithm,progress);
}
/// <summary>
/// <param name="blockSize">Block size used to calculate leaf hashes</param>
/// <param name="algorithm"></param>
/// <returns>A <see cref="TreeHash"/> with the block hashes and top hash</returns>
- public static TreeHash CalculateTreeHash(string filePath, int blockSize, string algorithm)
+ public static TreeHash CalculateTreeHash(string filePath, int blockSize, string algorithm,IProgress<double> progress )
{
if (String.IsNullOrWhiteSpace(filePath))
throw new ArgumentNullException("filePath");
if (String.IsNullOrWhiteSpace(algorithm))
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
- var hash=CalculateTreeHashAsync(filePath, blockSize, algorithm, 1);
+ var hash=CalculateTreeHashAsync(filePath, blockSize, algorithm, 1,progress);
return hash;
}
- public static TreeHash CalculateTreeHashAsync(FileInfo fileInfo, int blockSize, string algorithm, byte parallelism)
+ public static TreeHash CalculateTreeHashAsync(FileInfo fileInfo, int blockSize, string algorithm, byte parallelism,IProgress<double> progress )
{
if (fileInfo == null)
throw new ArgumentNullException("fileInfo");
throw new ArgumentNullException("algorithm");
Contract.EndContractBlock();
- return CalculateTreeHashAsync(fileInfo.FullName, blockSize, algorithm, parallelism);
+ return CalculateTreeHashAsync(fileInfo.FullName, blockSize, algorithm, parallelism,progress);
}
- public static TreeHash CalculateTreeHashAsync(string filePath, int blockSize,string algorithm, int parallelism)
+ public static TreeHash CalculateTreeHashAsync(string filePath, int blockSize,string algorithm, int parallelism,IProgress<double> progress )
{
if (String.IsNullOrWhiteSpace(filePath))
throw new ArgumentNullException("filePath");
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, true))
{
//Calculate the blocks asyncrhonously
- var hashes = BlockHashAlgorithms.CalculateBlockHashesInPlacePFor(stream, blockSize, algorithm, parallelism).Result;
+ var hashes = BlockHashAlgorithms.CalculateBlockHashesInPlacePFor(stream, blockSize, algorithm, parallelism,progress).Result;
//And then proceed with creating and returning a TreeHash
var length = stream.Length;
projects / pithos-ms-client / commitdiff