Revision 9d2c0fc0
b/trunk/Pithos.Client.WPF/Shell/ShellViewModel.cs | ||
---|---|---|
409 | 409 |
|
410 | 410 |
public void GoToSite(AccountInfo account) |
411 | 411 |
{ |
412 |
Process.Start(account.SiteUri); |
|
412 |
var uri = account.SiteUri.Replace("http://","https://"); |
|
413 |
Process.Start(uri); |
|
413 | 414 |
} |
414 | 415 |
|
415 |
/// <summary>
|
|
416 |
/// <summary>
|
|
416 | 417 |
/// Open an explorer window to the target path's directory |
417 | 418 |
/// and select the file |
418 | 419 |
/// </summary> |
b/trunk/Pithos.Core/Agents/NetworkAgent.cs | ||
---|---|---|
669 | 669 |
{ |
670 | 670 |
try |
671 | 671 |
{ |
672 |
|
|
672 | 673 |
var accountInfo = action.AccountInfo; |
673 | 674 |
|
674 | 675 |
var fileInfo = action.LocalFile; |
675 | 676 |
|
676 | 677 |
if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase)) |
677 | 678 |
return; |
679 |
//Do not upload files in conflict |
|
680 |
if (action.FileState.FileStatus == FileStatus.Conflict ) |
|
681 |
{ |
|
682 |
Log.InfoFormat("Skipping file in conflict [{0}]",fileInfo.FullName); |
|
683 |
return; |
|
684 |
} |
|
685 |
if (action.FileState.FileStatus == FileStatus.Forbidden) |
|
686 |
{ |
|
687 |
Log.InfoFormat("Skipping forbidden file [{0}]",fileInfo.FullName); |
|
688 |
return; |
|
689 |
} |
|
678 | 690 |
|
679 | 691 |
var relativePath = fileInfo.AsRelativeTo(accountInfo.AccountPath); |
680 | 692 |
if (relativePath.StartsWith(FolderConstants.OthersFolder)) |
... | ... | |
707 | 719 |
|
708 | 720 |
var cloudFile = action.CloudFile; |
709 | 721 |
var account = cloudFile.Account ?? accountInfo.UserName; |
722 |
try |
|
723 |
{ |
|
710 | 724 |
|
711 | 725 |
var client = new CloudFilesClient(accountInfo); |
712 | 726 |
//Even if GetObjectInfo times out, we can proceed with the upload |
... | ... | |
717 | 731 |
return; |
718 | 732 |
|
719 | 733 |
//TODO: Check how a directory hash is calculated -> All dirs seem to have the same hash |
720 |
if (fileInfo is DirectoryInfo) |
|
721 |
{ |
|
722 |
//If the directory doesn't exist the Hash property will be empty |
|
723 |
if (String.IsNullOrWhiteSpace(info.Hash)) |
|
724 |
//Go on and create the directory |
|
725 |
await |
|
726 |
client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName, |
|
727 |
String.Empty, "application/directory"); |
|
728 |
} |
|
729 |
else |
|
730 |
{ |
|
734 |
if (fileInfo is DirectoryInfo)
|
|
735 |
{
|
|
736 |
//If the directory doesn't exist the Hash property will be empty
|
|
737 |
if (String.IsNullOrWhiteSpace(info.Hash))
|
|
738 |
//Go on and create the directory
|
|
739 |
await
|
|
740 |
client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName,
|
|
741 |
String.Empty, "application/directory");
|
|
742 |
}
|
|
743 |
else
|
|
744 |
{
|
|
731 | 745 |
|
732 |
var cloudHash = info.Hash.ToLower(); |
|
746 |
var cloudHash = info.Hash.ToLower();
|
|
733 | 747 |
|
734 |
var hash = action.LocalHash.Value; |
|
735 |
var topHash = action.TopHash.Value; |
|
748 |
var hash = action.LocalHash.Value;
|
|
749 |
var topHash = action.TopHash.Value;
|
|
736 | 750 |
|
737 |
//If the file hashes match, abort the upload |
|
738 |
if (hash == cloudHash || topHash == cloudHash) |
|
739 |
{ |
|
740 |
//but store any metadata changes |
|
741 |
StatusKeeper.StoreInfo(fullFileName, info); |
|
742 |
Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName); |
|
743 |
return; |
|
744 |
} |
|
751 |
//If the file hashes match, abort the upload
|
|
752 |
if (hash == cloudHash || topHash == cloudHash)
|
|
753 |
{
|
|
754 |
//but store any metadata changes
|
|
755 |
StatusKeeper.StoreInfo(fullFileName, info);
|
|
756 |
Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
|
|
757 |
return;
|
|
758 |
}
|
|
745 | 759 |
|
746 | 760 |
|
747 |
//Mark the file as modified while we upload it |
|
748 |
StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified); |
|
749 |
//And then upload it |
|
761 |
//Mark the file as modified while we upload it
|
|
762 |
StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
|
|
763 |
//And then upload it
|
|
750 | 764 |
|
751 |
//Upload even small files using the Hashmap. The server may already contain |
|
752 |
//the relevant block |
|
765 |
//Upload even small files using the Hashmap. The server may already contain
|
|
766 |
//the relevant block
|
|
753 | 767 |
|
754 |
//First, calculate the tree hash |
|
755 |
var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize, |
|
756 |
accountInfo.BlockHash, 2); |
|
768 |
//First, calculate the tree hash
|
|
769 |
var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize,
|
|
770 |
accountInfo.BlockHash, 2);
|
|
757 | 771 |
|
758 |
await |
|
759 |
UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash); |
|
772 |
//TODO: If the upload fails with a 403, abort it and mark conflict |
|
773 |
|
|
774 |
await |
|
775 |
UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash); |
|
776 |
} |
|
777 |
//If everything succeeds, change the file and overlay status to normal |
|
778 |
StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal); |
|
779 |
} |
|
780 |
catch (WebException exc) |
|
781 |
{ |
|
782 |
var response=(exc.Response as HttpWebResponse); |
|
783 |
if (response.StatusCode == HttpStatusCode.Forbidden) |
|
784 |
{ |
|
785 |
StatusKeeper.SetFileState(fileInfo.FullName,FileStatus.Forbidden, FileOverlayStatus.Conflict); |
|
786 |
} |
|
760 | 787 |
} |
761 |
//If everything succeeds, change the file and overlay status to normal |
|
762 |
StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal); |
|
763 | 788 |
} |
764 | 789 |
//Notify the Shell to update the overlays |
765 | 790 |
NativeMethods.RaiseChangeNotification(fullFileName); |
b/trunk/Pithos.Core/IPithosWorkflow.cs | ||
---|---|---|
63 | 63 |
Renamed, |
64 | 64 |
Deleted, |
65 | 65 |
Conflict, |
66 |
Unversioned |
|
66 |
Unversioned, |
|
67 |
Forbidden |
|
67 | 68 |
} |
68 | 69 |
} |
b/trunk/Pithos.Network/AccountInfo.cs | ||
---|---|---|
76 | 76 |
|
77 | 77 |
} |
78 | 78 |
|
79 |
public string SiteUri { get; set; } |
|
79 |
private string _siteUri; |
|
80 |
public string SiteUri |
|
81 |
{ |
|
82 |
get { return _siteUri; } |
|
83 |
set |
|
84 |
{ |
|
85 |
_siteUri = value; |
|
86 |
} |
|
87 |
} |
|
80 | 88 |
|
81 | 89 |
public List<Group> Groups { get; set; } |
82 | 90 |
} |
b/trunk/Pithos.Network/BlockHashAlgorithms.cs | ||
---|---|---|
50 | 50 |
namespace Pithos.Network |
51 | 51 |
{ |
52 | 52 |
using System; |
53 |
using System.Collections.Generic; |
|
54 |
using System.Linq; |
|
55 |
using System.Text; |
|
56 | 53 |
|
57 | 54 |
/// <summary> |
58 | 55 |
/// TODO: Update summary. |
... | ... | |
61 | 58 |
{ |
62 | 59 |
private static readonly log4net.ILog Log = log4net.LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); |
63 | 60 |
|
61 |
/* |
|
64 | 62 |
public static Func<FileStream, int, string, ConcurrentDictionary<int, byte[]>, int, Task<ConcurrentDictionary<int, byte[]>>> CalculateBlockHash; |
65 | 63 |
|
66 | 64 |
public static Task<ConcurrentDictionary<int, byte[]>> CalculateBlockHashesRecursiveAsync(FileStream stream, int blockSize, string algorithm, ConcurrentDictionary<int, byte[]> hashes = null, int index = 0) |
... | ... | |
118 | 116 |
int read; |
119 | 117 |
while ((read = await stream.ReadAsync(buffer, 0, blockSize)) > 0) |
120 | 118 |
{ |
121 |
//TODO: identify the value of index |
|
122 |
|
|
123 | 119 |
using (var hasher = HashAlgorithm.Create(algorithm)) |
124 | 120 |
{ |
125 | 121 |
//This code was added for compatibility with the way Pithos calculates the last hash |
... | ... | |
130 | 126 |
hashes[index] = hash; |
131 | 127 |
} |
132 | 128 |
index += read; |
133 |
};
|
|
129 |
} |
|
134 | 130 |
return hashes; |
135 | 131 |
} |
136 | 132 |
|
... | ... | |
150 | 146 |
var size = stream.Length; |
151 | 147 |
Log.DebugFormat("Hashing [{0}] size [{1}]",path,size); |
152 | 148 |
|
153 |
/* |
|
154 | 149 |
var options = new ExecutionDataflowBlockOptions {BoundedCapacity = parallelism,MaxDegreeOfParallelism=parallelism}; |
155 | 150 |
var hashBlock=new ActionBlock<Tuple<int,byte[]>>(input=> |
156 | 151 |
{ |
... | ... | |
166 | 161 |
hashes[idx] = hash; |
167 | 162 |
} |
168 | 163 |
},options); |
169 |
*/ |
|
170 | 164 |
|
171 | 165 |
var buffer = new byte[blockSize]; |
172 | 166 |
int read; |
173 | 167 |
int index = 0; |
168 |
while ((read = await stream.ReadAsync(buffer, 0, blockSize)) > 0) |
|
169 |
{ |
|
170 |
var block = new byte[read]; |
|
171 |
Buffer.BlockCopy(buffer, 0, block, 0, read); |
|
172 |
await hashBlock.SendAsync(Tuple.Create(index, block)); |
|
173 |
index += read; |
|
174 |
} |
|
175 |
|
|
176 |
|
|
177 |
hashBlock.Complete(); |
|
178 |
await hashBlock.Completion; |
|
179 |
|
|
180 |
return hashes; |
|
181 |
} |
|
182 |
|
|
183 |
public static async Task<ConcurrentDictionary<int, byte[]>> CalculateBlockHashesInPlace(FileStream stream, int blockSize, string algorithm, int parallelism) |
|
184 |
{ |
|
185 |
if (stream == null) |
|
186 |
throw new ArgumentNullException("stream"); |
|
187 |
if (String.IsNullOrWhiteSpace(algorithm)) |
|
188 |
throw new ArgumentNullException("algorithm"); |
|
189 |
if (blockSize <= 0) |
|
190 |
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); |
|
191 |
Contract.EndContractBlock(); |
|
192 |
|
|
193 |
var hashes = new ConcurrentDictionary<int, byte[]>(); |
|
194 |
|
|
195 |
var path = stream.Name; |
|
196 |
var size = stream.Length; |
|
197 |
Log.DebugFormat("Hashing [{0}] size [{1}]",path,size); |
|
198 |
|
|
199 |
|
|
200 |
var buffer = new byte[blockSize]; |
|
201 |
var index = 0; |
|
174 | 202 |
using (var hasher = HashAlgorithm.Create(algorithm)) |
175 | 203 |
{ |
204 |
int read; |
|
176 | 205 |
while ((read = await stream.ReadAsync(buffer, 0, blockSize)) > 0) |
177 | 206 |
{ |
178 |
// var block = new byte[read]; |
|
179 |
|
|
180 | 207 |
//This code was added for compatibility with the way Pithos calculates the last hash |
181 | 208 |
//We calculate the hash only up to the last non-null byte |
182 | 209 |
var lastByteIndex = Array.FindLastIndex(buffer, read - 1, aByte => aByte != 0); |
... | ... | |
186 | 213 |
hashes[index] = hash; |
187 | 214 |
index += read; |
188 | 215 |
} |
216 |
} |
|
217 |
return hashes; |
|
218 |
} |
|
219 |
*/ |
|
220 |
|
|
221 |
public static async Task<ConcurrentDictionary<long, byte[]>> CalculateBlockHashesInPlacePFor(FileStream stream, int blockSize, string algorithm, int parallelism) |
|
222 |
{ |
|
223 |
if (stream == null) |
|
224 |
throw new ArgumentNullException("stream"); |
|
225 |
if (String.IsNullOrWhiteSpace(algorithm)) |
|
226 |
throw new ArgumentNullException("algorithm"); |
|
227 |
if (blockSize <= 0) |
|
228 |
throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero "); |
|
229 |
Contract.EndContractBlock(); |
|
230 |
|
|
231 |
var hashes = new ConcurrentDictionary<long, byte[]>(); |
|
232 |
|
|
233 |
var path = stream.Name; |
|
234 |
var size = stream.Length; |
|
235 |
Log.DebugFormat("Hashing [{0}] size [{1}]", path, size); |
|
189 | 236 |
|
190 |
/* |
|
191 |
Buffer.BlockCopy(buffer,0,block,0,read); |
|
192 |
await hashBlock.SendAsync(Tuple.Create(index, block)); |
|
193 |
*/ |
|
194 |
|
|
237 |
|
|
238 |
var buffer = new byte[parallelism][]; |
|
239 |
var hashers = new HashAlgorithm[parallelism]; |
|
240 |
for (var i = 0; i < parallelism; i++) |
|
241 |
{ |
|
242 |
buffer[i] = new byte[blockSize]; |
|
243 |
hashers[i] = HashAlgorithm.Create(algorithm); |
|
195 | 244 |
} |
196 |
|
|
245 |
try |
|
246 |
{ |
|
247 |
var indices = new long[parallelism]; |
|
248 |
var bufferCount = new int[parallelism]; |
|
197 | 249 |
|
198 |
/* |
|
199 |
hashBlock.Complete(); |
|
200 |
await hashBlock.Completion; |
|
201 |
*/ |
|
250 |
int read; |
|
251 |
int bufIdx = 0; |
|
252 |
long index = 0; |
|
253 |
while ((read = await stream.ReadAsync(buffer[bufIdx], 0, blockSize)) > 0) |
|
254 |
{ |
|
255 |
index += read; |
|
256 |
indices[bufIdx] = index; |
|
257 |
bufferCount[bufIdx] = read; |
|
258 |
//If we have filled the last buffer or if we have read from the last block, |
|
259 |
//we can calculate the clocks in parallel |
|
260 |
if (bufIdx == parallelism - 1 || read < blockSize) |
|
261 |
{ |
|
262 |
//var options = new ParallelOptions {MaxDegreeOfParallelism = parallelism}; |
|
263 |
Parallel.For(0, bufIdx + 1, idx => |
|
264 |
{ |
|
265 |
//This code was added for compatibility with the way Pithos calculates the last hash |
|
266 |
//We calculate the hash only up to the last non-null byte |
|
267 |
var lastByteIndex = Array.FindLastIndex(buffer[idx], |
|
268 |
bufferCount[idx] - 1, |
|
269 |
aByte => aByte != 0); |
|
270 |
|
|
271 |
var hasher = hashers[idx]; |
|
272 |
var hash = hasher.ComputeHash(buffer[idx], 0, lastByteIndex + 1); |
|
273 |
var filePosition = indices[idx]; |
|
274 |
/* |
|
275 |
Trace.TraceInformation("Hashed [{0}] [{1}/{2}] [{3:p}]", path, |
|
276 |
filePosition, size, |
|
277 |
(double)filePosition / size); |
|
278 |
*/ |
|
279 |
hashes[filePosition] = hash; |
|
280 |
}); |
|
281 |
} |
|
282 |
bufIdx = (bufIdx + 1) % parallelism; |
|
283 |
} |
|
284 |
} |
|
285 |
finally |
|
286 |
{ |
|
287 |
for (var i = 0; i < parallelism; i++) |
|
288 |
{ |
|
289 |
if (hashers[i] != null) |
|
290 |
hashers[i].Dispose(); |
|
291 |
} |
|
292 |
|
|
293 |
} |
|
202 | 294 |
|
203 | 295 |
return hashes; |
204 | 296 |
} |
205 | 297 |
|
206 | 298 |
static BlockHashAlgorithms() |
207 | 299 |
{ |
300 |
/* |
|
208 | 301 |
CalculateBlockHash = CalculateBlockHashesRecursiveAsync; |
302 |
*/ |
|
209 | 303 |
} |
210 | 304 |
} |
211 | 305 |
} |
b/trunk/Pithos.Network/Signature.cs | ||
---|---|---|
188 | 188 |
using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, true)) |
189 | 189 |
{ |
190 | 190 |
//Calculate the blocks asyncrhonously |
191 |
var hashes = await BlockHashAlgorithms.CalculateBlockHashesAgentAsync(stream, blockSize, algorithm, parallelism);
|
|
191 |
var hashes = await BlockHashAlgorithms.CalculateBlockHashesInPlacePFor(stream, blockSize, algorithm, parallelism);
|
|
192 | 192 |
|
193 | 193 |
//And then proceed with creating and returning a TreeHash |
194 | 194 |
var length = stream.Length; |
Also available in: Unified diff