Revision 9d2c0fc0

b/trunk/Pithos.Client.WPF/Shell/ShellViewModel.cs
409 409

  
410 410
		public void GoToSite(AccountInfo account)
411 411
		{
412
			Process.Start(account.SiteUri);
412
		    var uri = account.SiteUri.Replace("http://","https://");            
413
		    Process.Start(uri);
413 414
		}
414 415

  
415
        /// <summary>
416
	    /// <summary>
416 417
        /// Open an explorer window to the target path's directory
417 418
        /// and select the file
418 419
        /// </summary>
b/trunk/Pithos.Core/Agents/NetworkAgent.cs
669 669
            {
670 670
                try
671 671
                {
672

  
672 673
                    var accountInfo = action.AccountInfo;
673 674

  
674 675
                    var fileInfo = action.LocalFile;
675 676

  
676 677
                    if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))
677 678
                        return;
679
                    //Do not upload files in conflict
680
                    if (action.FileState.FileStatus == FileStatus.Conflict )
681
                    {
682
                        Log.InfoFormat("Skipping file in conflict [{0}]",fileInfo.FullName);
683
                        return;
684
                    }
685
                    if (action.FileState.FileStatus == FileStatus.Forbidden)
686
                    {
687
                        Log.InfoFormat("Skipping forbidden file [{0}]",fileInfo.FullName);
688
                        return;
689
                    }
678 690

  
679 691
                    var relativePath = fileInfo.AsRelativeTo(accountInfo.AccountPath);
680 692
                    if (relativePath.StartsWith(FolderConstants.OthersFolder))
......
707 719

  
708 720
                        var cloudFile = action.CloudFile;
709 721
                        var account = cloudFile.Account ?? accountInfo.UserName;
722
                        try
723
                        {
710 724

  
711 725
                        var client = new CloudFilesClient(accountInfo);
712 726
                        //Even if GetObjectInfo times out, we can proceed with the upload            
......
717 731
                            return;
718 732

  
719 733
                        //TODO: Check how a directory hash is calculated -> All dirs seem to have the same hash
720
                        if (fileInfo is DirectoryInfo)
721
                        {
722
                            //If the directory doesn't exist the Hash property will be empty
723
                            if (String.IsNullOrWhiteSpace(info.Hash))
724
                                //Go on and create the directory
725
                                await
726
                                    client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName,
727
                                                     String.Empty, "application/directory");
728
                        }
729
                        else
730
                        {
734
                            if (fileInfo is DirectoryInfo)
735
                            {
736
                                //If the directory doesn't exist the Hash property will be empty
737
                                if (String.IsNullOrWhiteSpace(info.Hash))
738
                                    //Go on and create the directory
739
                                    await
740
                                        client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName,
741
                                                         String.Empty, "application/directory");
742
                            }
743
                            else
744
                            {
731 745

  
732
                            var cloudHash = info.Hash.ToLower();
746
                                var cloudHash = info.Hash.ToLower();
733 747

  
734
                            var hash = action.LocalHash.Value;
735
                            var topHash = action.TopHash.Value;
748
                                var hash = action.LocalHash.Value;
749
                                var topHash = action.TopHash.Value;
736 750

  
737
                            //If the file hashes match, abort the upload
738
                            if (hash == cloudHash || topHash == cloudHash)
739
                            {
740
                                //but store any metadata changes 
741
                                StatusKeeper.StoreInfo(fullFileName, info);
742
                                Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
743
                                return;
744
                            }
751
                                //If the file hashes match, abort the upload
752
                                if (hash == cloudHash || topHash == cloudHash)
753
                                {
754
                                    //but store any metadata changes 
755
                                    StatusKeeper.StoreInfo(fullFileName, info);
756
                                    Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
757
                                    return;
758
                                }
745 759

  
746 760

  
747
                            //Mark the file as modified while we upload it
748
                            StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
749
                            //And then upload it
761
                                //Mark the file as modified while we upload it
762
                                StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
763
                                //And then upload it
750 764

  
751
                            //Upload even small files using the Hashmap. The server may already contain
752
                            //the relevant block
765
                                //Upload even small files using the Hashmap. The server may already contain
766
                                //the relevant block
753 767

  
754
                            //First, calculate the tree hash
755
                            var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize,
756
                                                                                  accountInfo.BlockHash, 2);
768
                                //First, calculate the tree hash
769
                                var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize,
770
                                                                                      accountInfo.BlockHash, 2);
757 771

  
758
                            await
759
                                UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash);
772
                                //TODO: If the upload fails with a 403, abort it and mark conflict
773

  
774
                                await
775
                                    UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash);
776
                            }
777
                            //If everything succeeds, change the file and overlay status to normal
778
                            StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal);
779
                        }
780
                        catch (WebException exc)
781
                        {
782
                            var response=(exc.Response as HttpWebResponse);
783
                            if (response.StatusCode == HttpStatusCode.Forbidden)
784
                            {
785
                                StatusKeeper.SetFileState(fileInfo.FullName,FileStatus.Forbidden, FileOverlayStatus.Conflict);                                
786
                            }
760 787
                        }
761
                        //If everything succeeds, change the file and overlay status to normal
762
                        StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal);
763 788
                    }
764 789
                    //Notify the Shell to update the overlays
765 790
                    NativeMethods.RaiseChangeNotification(fullFileName);
b/trunk/Pithos.Core/IPithosWorkflow.cs
63 63
        Renamed,
64 64
        Deleted,
65 65
        Conflict,
66
        Unversioned
66
        Unversioned,
67
        Forbidden
67 68
    }
68 69
}
b/trunk/Pithos.Network/AccountInfo.cs
76 76

  
77 77
        }
78 78

  
79
        public string SiteUri { get; set; }
79
        private string _siteUri;
80
        public string SiteUri
81
        {
82
            get { return _siteUri; }
83
            set
84
            {
85
                _siteUri = value;
86
            }
87
        }
80 88

  
81 89
        public List<Group> Groups { get; set; }
82 90
    }
b/trunk/Pithos.Network/BlockHashAlgorithms.cs
50 50
namespace Pithos.Network
51 51
{
52 52
    using System;
53
    using System.Collections.Generic;
54
    using System.Linq;
55
    using System.Text;
56 53

  
57 54
    /// <summary>
58 55
    /// TODO: Update summary.
......
61 58
    {
62 59
        private static readonly log4net.ILog Log = log4net.LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
63 60

  
61
/*
64 62
        public static Func<FileStream, int, string, ConcurrentDictionary<int, byte[]>, int, Task<ConcurrentDictionary<int, byte[]>>> CalculateBlockHash;
65 63

  
66 64
        public static Task<ConcurrentDictionary<int, byte[]>> CalculateBlockHashesRecursiveAsync(FileStream stream, int blockSize, string algorithm, ConcurrentDictionary<int, byte[]> hashes = null, int index = 0)
......
118 116
            int read;
119 117
            while ((read = await stream.ReadAsync(buffer, 0, blockSize)) > 0)
120 118
            {
121
                //TODO: identify the value of index
122

  
123 119
                using (var hasher = HashAlgorithm.Create(algorithm))
124 120
                {
125 121
                    //This code was added for compatibility with the way Pithos calculates the last hash
......
130 126
                    hashes[index] = hash;
131 127
                }
132 128
                index += read;
133
            };
129
            }
134 130
            return hashes;
135 131
        }
136 132
        
......
150 146
            var size = stream.Length;
151 147
            Log.DebugFormat("Hashing [{0}] size [{1}]",path,size);
152 148
            
153
/*
154 149
            var options = new ExecutionDataflowBlockOptions {BoundedCapacity = parallelism,MaxDegreeOfParallelism=parallelism};
155 150
            var hashBlock=new ActionBlock<Tuple<int,byte[]>>(input=>
156 151
                              {
......
166 161
                                      hashes[idx] = hash;
167 162
                                  }                                  
168 163
                              },options);
169
*/
170 164

  
171 165
            var buffer = new byte[blockSize];
172 166
            int read;
173 167
            int index = 0;
168
            while ((read = await stream.ReadAsync(buffer, 0, blockSize)) > 0)
169
            {
170
                var block = new byte[read];
171
                Buffer.BlockCopy(buffer, 0, block, 0, read);
172
                await hashBlock.SendAsync(Tuple.Create(index, block));
173
                index += read;
174
            }
175
            
176

  
177
            hashBlock.Complete();
178
            await hashBlock.Completion;
179

  
180
            return hashes;
181
        } 
182
        
183
        public static async Task<ConcurrentDictionary<int, byte[]>> CalculateBlockHashesInPlace(FileStream stream, int blockSize, string algorithm, int parallelism)
184
        {
185
            if (stream == null)
186
                throw new ArgumentNullException("stream");
187
            if (String.IsNullOrWhiteSpace(algorithm))
188
                throw new ArgumentNullException("algorithm");
189
            if (blockSize <= 0)
190
                throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
191
            Contract.EndContractBlock();
192

  
193
            var hashes = new ConcurrentDictionary<int, byte[]>();
194

  
195
            var path = stream.Name;
196
            var size = stream.Length;
197
            Log.DebugFormat("Hashing [{0}] size [{1}]",path,size);
198
            
199

  
200
            var buffer = new byte[blockSize];
201
            var index = 0;
174 202
            using (var hasher = HashAlgorithm.Create(algorithm))
175 203
            {
204
                int read;
176 205
                while ((read = await stream.ReadAsync(buffer, 0, blockSize)) > 0)
177 206
                {
178
                    //                var block = new byte[read];
179

  
180 207
                    //This code was added for compatibility with the way Pithos calculates the last hash
181 208
                    //We calculate the hash only up to the last non-null byte
182 209
                    var lastByteIndex = Array.FindLastIndex(buffer, read - 1, aByte => aByte != 0);
......
186 213
                    hashes[index] = hash;
187 214
                    index += read;
188 215
                }
216
            }
217
            return hashes;
218
        }
219
*/
220

  
221
        public static async Task<ConcurrentDictionary<long, byte[]>> CalculateBlockHashesInPlacePFor(FileStream stream, int blockSize, string algorithm, int parallelism)
222
        {
223
            if (stream == null)
224
                throw new ArgumentNullException("stream");
225
            if (String.IsNullOrWhiteSpace(algorithm))
226
                throw new ArgumentNullException("algorithm");
227
            if (blockSize <= 0)
228
                throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
229
            Contract.EndContractBlock();
230

  
231
            var hashes = new ConcurrentDictionary<long, byte[]>();
232

  
233
            var path = stream.Name;
234
            var size = stream.Length;
235
            Log.DebugFormat("Hashing [{0}] size [{1}]", path, size);
189 236

  
190
                /*
191
                                Buffer.BlockCopy(buffer,0,block,0,read);
192
                                await hashBlock.SendAsync(Tuple.Create(index, block));
193
                */
194
                
237

  
238
            var buffer = new byte[parallelism][];
239
            var hashers = new HashAlgorithm[parallelism];
240
            for (var i = 0; i < parallelism; i++)
241
            {
242
                buffer[i] = new byte[blockSize];
243
                hashers[i] = HashAlgorithm.Create(algorithm);
195 244
            }
196
            
245
            try
246
            {
247
                var indices = new long[parallelism];
248
                var bufferCount = new int[parallelism];
197 249

  
198
/*
199
            hashBlock.Complete();
200
            await hashBlock.Completion;
201
*/
250
                int read;
251
                int bufIdx = 0;
252
                long index = 0;
253
                while ((read = await stream.ReadAsync(buffer[bufIdx], 0, blockSize)) > 0)
254
                {
255
                    index += read;
256
                    indices[bufIdx] = index;
257
                    bufferCount[bufIdx] = read;
258
                    //If we have filled the last buffer or if we have read from the last block,
259
                    //we can calculate the clocks in parallel
260
                    if (bufIdx == parallelism - 1 || read < blockSize)
261
                    {
262
                        //var options = new ParallelOptions {MaxDegreeOfParallelism = parallelism};
263
                        Parallel.For(0, bufIdx + 1, idx =>
264
                        {
265
                            //This code was added for compatibility with the way Pithos calculates the last hash
266
                            //We calculate the hash only up to the last non-null byte
267
                            var lastByteIndex = Array.FindLastIndex(buffer[idx],
268
                                                                    bufferCount[idx] - 1,
269
                                                                    aByte => aByte != 0);
270

  
271
                            var hasher = hashers[idx];
272
                            var hash = hasher.ComputeHash(buffer[idx], 0, lastByteIndex + 1);
273
                            var filePosition = indices[idx];
274
                            /*
275
                                                        Trace.TraceInformation("Hashed [{0}] [{1}/{2}] [{3:p}]", path,
276
                                                                                filePosition, size,
277
                                                                                (double)filePosition / size);
278
                            */
279
                            hashes[filePosition] = hash;
280
                        });
281
                    }
282
                    bufIdx = (bufIdx + 1) % parallelism;
283
                }
284
            }
285
            finally
286
            {
287
                for (var i = 0; i < parallelism; i++)
288
                {
289
                    if (hashers[i] != null)
290
                        hashers[i].Dispose();
291
                }
292

  
293
            }
202 294

  
203 295
            return hashes;
204 296
        }
205 297

  
206 298
        static BlockHashAlgorithms()
207 299
        {
300
/*
208 301
            CalculateBlockHash = CalculateBlockHashesRecursiveAsync;
302
*/
209 303
        }
210 304
    }
211 305
}
b/trunk/Pithos.Network/Signature.cs
188 188
            using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, blockSize, true))
189 189
            {
190 190
                //Calculate the blocks asyncrhonously
191
                var hashes = await BlockHashAlgorithms.CalculateBlockHashesAgentAsync(stream, blockSize, algorithm, parallelism);                
191
                var hashes = await BlockHashAlgorithms.CalculateBlockHashesInPlacePFor(stream, blockSize, algorithm, parallelism);                
192 192

  
193 193
                //And then proceed with creating and returning a TreeHash
194 194
                var length = stream.Length;

Also available in: Unified diff