Statistics
| Branch: | Revision:

root / trunk / Pithos.Core / Agents / NetworkAgent.cs @ aa7ac00e

History | View | Annotate | Download (36.2 kB)

1
#region
2
/* -----------------------------------------------------------------------
3
 * <copyright file="NetworkAgent.cs" company="GRNet">
4
 * 
5
 * Copyright 2011-2012 GRNET S.A. All rights reserved.
6
 *
7
 * Redistribution and use in source and binary forms, with or
8
 * without modification, are permitted provided that the following
9
 * conditions are met:
10
 *
11
 *   1. Redistributions of source code must retain the above
12
 *      copyright notice, this list of conditions and the following
13
 *      disclaimer.
14
 *
15
 *   2. Redistributions in binary form must reproduce the above
16
 *      copyright notice, this list of conditions and the following
17
 *      disclaimer in the documentation and/or other materials
18
 *      provided with the distribution.
19
 *
20
 *
21
 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32
 * POSSIBILITY OF SUCH DAMAGE.
33
 *
34
 * The views and conclusions contained in the software and
35
 * documentation are those of the authors and should not be
36
 * interpreted as representing official policies, either expressed
37
 * or implied, of GRNET S.A.
38
 * </copyright>
39
 * -----------------------------------------------------------------------
40
 */
41
#endregion
42

    
43
//TODO: Now there is a UUID tag. This can be used for renames/moves
44

    
45

    
46
using System;
47
using System.Collections.Concurrent;
48
using System.Collections.Generic;
49
using System.ComponentModel.Composition;
50
using System.Diagnostics;
51
using System.Diagnostics.Contracts;
52
using System.IO;
53
using System.Linq;
54
using System.Net;
55
using System.Threading;
56
using System.Threading.Tasks;
57
using System.Threading.Tasks.Dataflow;
58
using Castle.ActiveRecord;
59
using Pithos.Interfaces;
60
using Pithos.Network;
61
using log4net;
62

    
63
namespace Pithos.Core.Agents
64
{
65
    //TODO: Ensure all network operations use exact casing. Pithos is case sensitive
66
    [Export]
67
    public class NetworkAgent
68
    {
69
        private Agent<CloudAction> _agent;
70

    
71
        [System.ComponentModel.Composition.Import]
72
        private DeleteAgent _deleteAgent=new DeleteAgent();
73

    
74
        [System.ComponentModel.Composition.Import]
75
        public IStatusKeeper StatusKeeper { get; set; }
76
        
77
        public IStatusNotification StatusNotification { get; set; }
78

    
79
        private static readonly ILog Log = LogManager.GetLogger("NetworkAgent");
80

    
81
        private readonly ConcurrentBag<AccountInfo> _accounts = new ConcurrentBag<AccountInfo>();
82

    
83
        [System.ComponentModel.Composition.Import]
84
        public IPithosSettings Settings { get; set; }
85

    
86
        //The Pause event stops the poll agent to give priority to the network agent
87
        //Initially the event is signalled because we don't need to pause
88
        private readonly AsyncManualResetEvent _pauseEvent = new AsyncManualResetEvent(true);
89

    
90
        public AsyncManualResetEvent PauseEvent
91
        {
92
            get { return _pauseEvent; }
93
        }
94

    
95

    
96
        public void Start()
97
        {
98
            _agent = Agent<CloudAction>.Start(inbox =>
99
            {
100
                Action loop = null;
101
                loop = () =>
102
                {
103
                    _deleteAgent.PauseEvent.Wait();
104
                    var message = inbox.Receive();
105
                    var process=message.Then(Process,inbox.CancellationToken);
106
                    inbox.LoopAsync(process, loop);
107
                };
108
                loop();
109
            });
110

    
111
        }
112

    
113
        private async Task Process(CloudAction action)
114
        {
115
            if (action == null)
116
                throw new ArgumentNullException("action");
117
            if (action.AccountInfo==null)
118
                throw new ArgumentException("The action.AccountInfo is empty","action");
119
            Contract.EndContractBlock();
120

    
121

    
122

    
123

    
124
            using (log4net.ThreadContext.Stacks["NETWORK"].Push("PROCESS"))
125
            {                
126
                Log.InfoFormat("[ACTION] Start Processing {0}", action);
127

    
128
                var cloudFile = action.CloudFile;
129
                var downloadPath = action.GetDownloadPath();
130

    
131
                try
132
                {
133
                    _pauseEvent.Reset();
134
                    UpdateStatus(PithosStatus.Syncing);
135
                    var accountInfo = action.AccountInfo;
136

    
137
                    if (action.Action == CloudActionType.DeleteCloud)
138
                    {                        
139
                        //Redirect deletes to the delete agent 
140
                        _deleteAgent.Post((CloudDeleteAction)action);
141
                    }
142
                    if (_deleteAgent.IsDeletedFile(action))
143
                    {
144
                        //Clear the status of already deleted files to avoid reprocessing
145
                        if (action.LocalFile != null)
146
                            this.StatusKeeper.ClearFileStatus(action.LocalFile.FullName);
147
                    }
148
                    else
149
                    {
150
                        switch (action.Action)
151
                        {
152
                            case CloudActionType.UploadUnconditional:
153
                                //Abort if the file was deleted before we reached this point
154
                                await UploadCloudFile(action);
155
                                break;
156
                            case CloudActionType.DownloadUnconditional:
157
                                await DownloadCloudFile(accountInfo, cloudFile, downloadPath);
158
                                break;
159
                            case CloudActionType.RenameCloud:
160
                                var moveAction = (CloudMoveAction)action;
161
                                RenameCloudFile(accountInfo, moveAction);
162
                                break;
163
                            case CloudActionType.MustSynch:
164
                                if (!File.Exists(downloadPath) && !Directory.Exists(downloadPath))
165
                                {
166
                                    await DownloadCloudFile(accountInfo, cloudFile, downloadPath);
167
                                }
168
                                else
169
                                {
170
                                    await SyncFiles(accountInfo, action);
171
                                }
172
                                break;
173
                        }
174
                    }
175
                    Log.InfoFormat("[ACTION] End Processing {0}:{1}->{2}", action.Action, action.LocalFile,
176
                                           action.CloudFile.Name);
177
                }
178
                catch (WebException exc)
179
                {
180
                    Log.ErrorFormat("[WEB ERROR] {0} : {1} -> {2} due to exception\r\n{3}", action.Action, action.LocalFile, action.CloudFile, exc);
181
                }
182
                catch (OperationCanceledException)
183
                {
184
                    throw;
185
                }
186
                catch (DirectoryNotFoundException)
187
                {
188
                    Log.ErrorFormat("{0} : {1} -> {2}  failed because the directory was not found.\n Rescheduling a delete",
189
                        action.Action, action.LocalFile, action.CloudFile);
190
                    //Post a delete action for the missing file
191
                    Post(new CloudDeleteAction(action));
192
                }
193
                catch (FileNotFoundException)
194
                {
195
                    Log.ErrorFormat("{0} : {1} -> {2}  failed because the file was not found.\n Rescheduling a delete",
196
                        action.Action, action.LocalFile, action.CloudFile);
197
                    //Post a delete action for the missing file
198
                    Post(new CloudDeleteAction(action));
199
                }
200
                catch (Exception exc)
201
                {
202
                    Log.ErrorFormat("[REQUEUE] {0} : {1} -> {2} due to exception\r\n{3}",
203
                                     action.Action, action.LocalFile, action.CloudFile, exc);
204

    
205
                    _agent.Post(action);
206
                }
207
                finally
208
                {
209
                    if (_agent.IsEmpty)
210
                        _pauseEvent.Set();
211
                    UpdateStatus(PithosStatus.InSynch);                                        
212
                }
213
            }
214
        }
215

    
216
        private void UpdateStatus(PithosStatus status)
217
        {
218
            StatusKeeper.SetPithosStatus(status);
219
            StatusNotification.Notify(new Notification());
220
        }
221

    
222
        
223
        private async Task SyncFiles(AccountInfo accountInfo,CloudAction action)
224
        {
225
            if (accountInfo == null)
226
                throw new ArgumentNullException("accountInfo");
227
            if (action==null)
228
                throw new ArgumentNullException("action");
229
            if (action.LocalFile==null)
230
                throw new ArgumentException("The action's local file is not specified","action");
231
            if (!Path.IsPathRooted(action.LocalFile.FullName))
232
                throw new ArgumentException("The action's local file path must be absolute","action");
233
            if (action.CloudFile== null)
234
                throw new ArgumentException("The action's cloud file is not specified", "action");
235
            Contract.EndContractBlock();
236

    
237
            var localFile = action.LocalFile;
238
            var cloudFile = action.CloudFile;
239
            var downloadPath=action.LocalFile.GetProperCapitalization();
240

    
241
            var cloudHash = cloudFile.Hash.ToLower();
242
            var localHash = action.LocalHash.Value.ToLower();
243
            var topHash = action.TopHash.Value.ToLower();
244

    
245
            //Not enough to compare only the local hashes, also have to compare the tophashes
246
            
247
            //If any of the hashes match, we are done
248
            if ((cloudHash == localHash || cloudHash == topHash))
249
            {
250
                Log.InfoFormat("Skipping {0}, hashes match",downloadPath);
251
                return;
252
            }
253

    
254
            //The hashes DON'T match. We need to sync
255
            var lastLocalTime = localFile.LastWriteTime;
256
            var lastUpTime = cloudFile.Last_Modified;
257
            
258
            //If the local file is newer upload it
259
            if (lastUpTime <= lastLocalTime)
260
            {
261
                //It probably means it was changed while the app was down                        
262
                UploadCloudFile(action);
263
            }
264
            else
265
            {
266
                //It the cloud file has a later date, it was modified by another user or computer.
267
                //We need to check the local file's status                
268
                var status = StatusKeeper.GetFileStatus(downloadPath);
269
                switch (status)
270
                {
271
                    case FileStatus.Unchanged:                        
272
                        //If the local file's status is Unchanged, we can go on and download the newer cloud file
273
                        await DownloadCloudFile(accountInfo,cloudFile,downloadPath);
274
                        break;
275
                    case FileStatus.Modified:
276
                        //If the local file is Modified, we may have a conflict. In this case we should mark the file as Conflict
277
                        //We can't ensure that a file modified online since the last time will appear as Modified, unless we 
278
                        //index all files before we start listening.                       
279
                    case FileStatus.Created:
280
                        //If the local file is Created, it means that the local and cloud files aren't related,
281
                        // yet they have the same name.
282

    
283
                        //In both cases we must mark the file as in conflict
284
                        ReportConflict(downloadPath);
285
                        break;
286
                    default:
287
                        //Other cases should never occur. Mark them as Conflict as well but log a warning
288
                        ReportConflict(downloadPath);
289
                        Log.WarnFormat("Unexcepted status {0} for file {1}->{2}", status,
290
                                       downloadPath, action.CloudFile.Name);
291
                        break;
292
                }
293
            }
294
        }
295

    
296
        private void ReportConflict(string downloadPath)
297
        {
298
            if (String.IsNullOrWhiteSpace(downloadPath))
299
                throw new ArgumentNullException("downloadPath");
300
            Contract.EndContractBlock();
301

    
302
            StatusKeeper.SetFileOverlayStatus(downloadPath, FileOverlayStatus.Conflict);
303
            UpdateStatus(PithosStatus.HasConflicts);
304
            var message = String.Format("Conflict detected for file {0}", downloadPath);
305
            Log.Warn(message);
306
            StatusNotification.NotifyChange(message, TraceLevel.Warning);
307
        }
308

    
309
        public void Post(CloudAction cloudAction)
310
        {
311
            if (cloudAction == null)
312
                throw new ArgumentNullException("cloudAction");
313
            if (cloudAction.AccountInfo==null)
314
                throw new ArgumentException("The CloudAction.AccountInfo is empty","cloudAction");
315
            Contract.EndContractBlock();
316

    
317
            _deleteAgent.PauseEvent.Wait();
318

    
319
            //If the action targets a local file, add a treehash calculation
320
            if (!(cloudAction is CloudDeleteAction) && cloudAction.LocalFile as FileInfo != null)
321
            {
322
                var accountInfo = cloudAction.AccountInfo;
323
                var localFile = (FileInfo) cloudAction.LocalFile;
324
                if (localFile.Length > accountInfo.BlockSize)
325
                    cloudAction.TopHash =
326
                        new Lazy<string>(() => Signature.CalculateTreeHashAsync(localFile,
327
                                                                                accountInfo.BlockSize,
328
                                                                                accountInfo.BlockHash, Settings.HashingParallelism).Result
329
                                                    .TopHash.ToHashString());
330
                else
331
                {
332
                    cloudAction.TopHash = new Lazy<string>(() => cloudAction.LocalHash.Value);
333
                }
334
            }
335
            else
336
            {
337
                //The hash for a directory is the empty string
338
                cloudAction.TopHash = new Lazy<string>(() => String.Empty);
339
            }
340
            
341
            if (cloudAction is CloudDeleteAction)
342
                _deleteAgent.Post((CloudDeleteAction)cloudAction);
343
            else
344
                _agent.Post(cloudAction);
345
        }
346
       
347

    
348
        public IEnumerable<CloudAction> GetEnumerable()
349
        {
350
            return _agent.GetEnumerable();
351
        }
352

    
353
        public Task GetDeleteAwaiter()
354
        {
355
            return _deleteAgent.PauseEvent.WaitAsync();
356
        }
357
        public CancellationToken CancellationToken
358
        {
359
            get { return _agent.CancellationToken; }
360
        }
361

    
362
        private static FileAgent GetFileAgent(AccountInfo accountInfo)
363
        {
364
            return AgentLocator<FileAgent>.Get(accountInfo.AccountPath);
365
        }
366

    
367

    
368

    
369
        private void RenameCloudFile(AccountInfo accountInfo,CloudMoveAction action)
370
        {
371
            if (accountInfo==null)
372
                throw new ArgumentNullException("accountInfo");
373
            if (action==null)
374
                throw new ArgumentNullException("action");
375
            if (action.CloudFile==null)
376
                throw new ArgumentException("CloudFile","action");
377
            if (action.LocalFile==null)
378
                throw new ArgumentException("LocalFile","action");
379
            if (action.OldLocalFile==null)
380
                throw new ArgumentException("OldLocalFile","action");
381
            if (action.OldCloudFile==null)
382
                throw new ArgumentException("OldCloudFile","action");
383
            Contract.EndContractBlock();
384
            
385
            
386
            var newFilePath = action.LocalFile.FullName;
387
            
388
            //How do we handle concurrent renames and deletes/uploads/downloads?
389
            //* A conflicting upload means that a file was renamed before it had a chance to finish uploading
390
            //  This should never happen as the network agent executes only one action at a time
391
            //* A conflicting download means that the file was modified on the cloud. While we can go on and complete
392
            //  the rename, there may be a problem if the file is downloaded in blocks, as subsequent block requests for the 
393
            //  same name will fail.
394
            //  This should never happen as the network agent executes only one action at a time.
395
            //* A conflicting delete can happen if the rename was followed by a delete action that didn't have the chance
396
            //  to remove the rename from the queue.
397
            //  We can probably ignore this case. It will result in an error which should be ignored            
398

    
399
            
400
            //The local file is already renamed
401
            StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Modified);
402

    
403

    
404
            var account = action.CloudFile.Account ?? accountInfo.UserName;
405
            var container = action.CloudFile.Container;
406
            
407
            var client = new CloudFilesClient(accountInfo);
408
            //TODO: What code is returned when the source file doesn't exist?
409
            client.MoveObject(account, container, action.OldCloudFile.Name, container, action.CloudFile.Name);
410

    
411
            StatusKeeper.SetFileStatus(newFilePath, FileStatus.Unchanged);
412
            StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Normal);
413
            NativeMethods.RaiseChangeNotification(newFilePath);
414
        }
415

    
416
        //Download a file.
417
        private async Task DownloadCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile , string filePath)
418
        {
419
            if (accountInfo == null)
420
                throw new ArgumentNullException("accountInfo");
421
            if (cloudFile == null)
422
                throw new ArgumentNullException("cloudFile");
423
            if (String.IsNullOrWhiteSpace(cloudFile.Account))
424
                throw new ArgumentNullException("cloudFile");
425
            if (String.IsNullOrWhiteSpace(cloudFile.Container))
426
                throw new ArgumentNullException("cloudFile");
427
            if (String.IsNullOrWhiteSpace(filePath))
428
                throw new ArgumentNullException("filePath");
429
            if (!Path.IsPathRooted(filePath))
430
                throw new ArgumentException("The filePath must be rooted", "filePath");
431
            Contract.EndContractBlock();
432
            
433

    
434
            var localPath = Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath);
435
            var relativeUrl = new Uri(cloudFile.Name, UriKind.Relative);
436

    
437
            var url = relativeUrl.ToString();
438
            if (cloudFile.Name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase))
439
                return;
440

    
441

    
442
            //Are we already downloading or uploading the file? 
443
            using (var gate=NetworkGate.Acquire(localPath, NetworkOperation.Downloading))
444
            {
445
                if (gate.Failed)
446
                    return;
447
                
448
                var client = new CloudFilesClient(accountInfo);
449
                var account = cloudFile.Account;
450
                var container = cloudFile.Container;
451

    
452
                if (cloudFile.Content_Type == @"application/directory")
453
                {
454
                    if (!Directory.Exists(localPath))
455
                        Directory.CreateDirectory(localPath);
456
                }
457
                else
458
                {                    
459
                    //Retrieve the hashmap from the server
460
                    var serverHash = await client.GetHashMap(account, container, url);
461
                    //If it's a small file
462
                    if (serverHash.Hashes.Count == 1)
463
                        //Download it in one go
464
                        await
465
                            DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath, serverHash);
466
                        //Otherwise download it block by block
467
                    else
468
                        await DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath, serverHash);
469

    
470
                    if (cloudFile.AllowedTo == "read")
471
                    {
472
                        var attributes = File.GetAttributes(localPath);
473
                        File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly);                        
474
                    }
475
                }
476

    
477
                //Now we can store the object's metadata without worrying about ghost status entries
478
                StatusKeeper.StoreInfo(localPath, cloudFile);
479
                
480
            }
481
        }
482

    
483
        //Download a small file with a single GET operation
484
        private async Task DownloadEntireFileAsync(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath,TreeHash serverHash)
485
        {
486
            if (client == null)
487
                throw new ArgumentNullException("client");
488
            if (cloudFile==null)
489
                throw new ArgumentNullException("cloudFile");
490
            if (relativeUrl == null)
491
                throw new ArgumentNullException("relativeUrl");
492
            if (String.IsNullOrWhiteSpace(filePath))
493
                throw new ArgumentNullException("filePath");
494
            if (!Path.IsPathRooted(filePath))
495
                throw new ArgumentException("The localPath must be rooted", "filePath");
496
            Contract.EndContractBlock();
497

    
498
            var localPath = Pithos.Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath);
499
            //If the file already exists
500
            if (File.Exists(localPath))
501
            {
502
                //First check with MD5 as this is a small file
503
                var localMD5 = Signature.CalculateMD5(localPath);
504
                var cloudHash=serverHash.TopHash.ToHashString();
505
                if (localMD5==cloudHash)
506
                    return;
507
                //Then check with a treehash
508
                var localTreeHash = Signature.CalculateTreeHash(localPath, serverHash.BlockSize, serverHash.BlockHash);
509
                var localHash = localTreeHash.TopHash.ToHashString();
510
                if (localHash==cloudHash)
511
                    return;
512
            }
513
            StatusNotification.Notify(new CloudNotification { Data = cloudFile });
514

    
515
            var fileAgent = GetFileAgent(accountInfo);
516
            //Calculate the relative file path for the new file
517
            var relativePath = relativeUrl.RelativeUriToFilePath();
518
            //The file will be stored in a temporary location while downloading with an extension .download
519
            var tempPath = Path.Combine(fileAgent.CachePath, relativePath + ".download");
520
            //Make sure the target folder exists. DownloadFileTask will not create the folder
521
            var tempFolder = Path.GetDirectoryName(tempPath);
522
            if (!Directory.Exists(tempFolder))
523
                Directory.CreateDirectory(tempFolder);
524

    
525
            //Download the object to the temporary location
526
            await client.GetObject(cloudFile.Account, cloudFile.Container, relativeUrl.ToString(), tempPath);
527

    
528
            //Create the local folder if it doesn't exist (necessary for shared objects)
529
            var localFolder = Path.GetDirectoryName(localPath);
530
            if (!Directory.Exists(localFolder))
531
                Directory.CreateDirectory(localFolder);            
532
            //And move it to its actual location once downloading is finished
533
            if (File.Exists(localPath))
534
                File.Replace(tempPath,localPath,null,true);
535
            else
536
                File.Move(tempPath,localPath);
537
            //Notify listeners that a local file has changed
538
            StatusNotification.NotifyChangedFile(localPath);
539

    
540
                       
541
        }
542

    
543
        //Download a file asynchronously using blocks
544
        public async Task DownloadWithBlocks(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath, TreeHash serverHash)
545
        {
546
            if (client == null)
547
                throw new ArgumentNullException("client");
548
            if (cloudFile == null)
549
                throw new ArgumentNullException("cloudFile");
550
            if (relativeUrl == null)
551
                throw new ArgumentNullException("relativeUrl");
552
            if (String.IsNullOrWhiteSpace(filePath))
553
                throw new ArgumentNullException("filePath");
554
            if (!Path.IsPathRooted(filePath))
555
                throw new ArgumentException("The filePath must be rooted", "filePath");
556
            if (serverHash == null)
557
                throw new ArgumentNullException("serverHash");
558
            Contract.EndContractBlock();
559
            
560
           var fileAgent = GetFileAgent(accountInfo);
561
            var localPath = Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath);
562
            
563
            //Calculate the relative file path for the new file
564
            var relativePath = relativeUrl.RelativeUriToFilePath();
565
            var blockUpdater = new BlockUpdater(fileAgent.CachePath, localPath, relativePath, serverHash);
566

    
567
            
568
                        
569
            //Calculate the file's treehash
570
            var treeHash = await Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize, serverHash.BlockHash, 2);
571
                
572
            //And compare it with the server's hash
573
            var upHashes = serverHash.GetHashesAsStrings();
574
            var localHashes = treeHash.HashDictionary;
575
            for (int i = 0; i < upHashes.Length; i++)
576
            {
577
                //For every non-matching hash
578
                var upHash = upHashes[i];
579
                if (!localHashes.ContainsKey(upHash))
580
                {
581
                    StatusNotification.Notify(new CloudNotification { Data = cloudFile });
582

    
583
                    if (blockUpdater.UseOrphan(i, upHash))
584
                    {
585
                        Log.InfoFormat("[BLOCK GET] ORPHAN FOUND for {0} of {1} for {2}", i, upHashes.Length, localPath);
586
                        continue;
587
                    }
588
                    Log.InfoFormat("[BLOCK GET] START {0} of {1} for {2}", i, upHashes.Length, localPath);
589
                    var start = i*serverHash.BlockSize;
590
                    //To download the last block just pass a null for the end of the range
591
                    long? end = null;
592
                    if (i < upHashes.Length - 1 )
593
                        end= ((i + 1)*serverHash.BlockSize) ;
594
                            
595
                    //Download the missing block
596
                    var block = await client.GetBlock(cloudFile.Account, cloudFile.Container, relativeUrl, start, end);
597

    
598
                    //and store it
599
                    blockUpdater.StoreBlock(i, block);
600

    
601

    
602
                    Log.InfoFormat("[BLOCK GET] FINISH {0} of {1} for {2}", i, upHashes.Length, localPath);
603
                }
604
            }
605

    
606
            //Want to avoid notifications if no changes were made
607
            var hasChanges = blockUpdater.HasBlocks;
608
            blockUpdater.Commit();
609
            
610
            if (hasChanges)
611
                //Notify listeners that a local file has changed
612
                StatusNotification.NotifyChangedFile(localPath);
613

    
614
            Log.InfoFormat("[BLOCK GET] COMPLETE {0}", localPath);            
615
        }
616

    
617

    
618
        private async Task UploadCloudFile(CloudAction action)
619
        {
620
            if (action == null)
621
                throw new ArgumentNullException("action");           
622
            Contract.EndContractBlock();
623

    
624
            try
625
            {                
626
                var accountInfo = action.AccountInfo;
627

    
628
                var fileInfo = action.LocalFile;
629

    
630
                if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))
631
                    return;
632
                
633
                var relativePath = fileInfo.AsRelativeTo(accountInfo.AccountPath);
634
                if (relativePath.StartsWith(FolderConstants.OthersFolder))
635
                {
636
                    var parts = relativePath.Split('\\');
637
                    var accountName = parts[1];
638
                    var oldName = accountInfo.UserName;
639
                    var absoluteUri = accountInfo.StorageUri.AbsoluteUri;
640
                    var nameIndex = absoluteUri.IndexOf(oldName);
641
                    var root = absoluteUri.Substring(0, nameIndex);
642

    
643
                    accountInfo = new AccountInfo
644
                    {
645
                        UserName = accountName,
646
                        AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
647
                        StorageUri = new Uri(root + accountName),
648
                        BlockHash = accountInfo.BlockHash,
649
                        BlockSize = accountInfo.BlockSize,
650
                        Token = accountInfo.Token
651
                    };
652
                }
653

    
654

    
655
                var fullFileName = fileInfo.GetProperCapitalization();
656
                using (var gate = NetworkGate.Acquire(fullFileName, NetworkOperation.Uploading))
657
                {
658
                    //Abort if the file is already being uploaded or downloaded
659
                    if (gate.Failed)
660
                        return;
661

    
662
                    var cloudFile = action.CloudFile;
663
                    var account = cloudFile.Account ?? accountInfo.UserName;
664

    
665
                    var client = new CloudFilesClient(accountInfo);                    
666
                    //Even if GetObjectInfo times out, we can proceed with the upload            
667
                    var info = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
668

    
669
                    //If this is a read-only file, do not upload changes
670
                    if (info.AllowedTo == "read")
671
                        return;
672
                    
673
                    //TODO: Check how a directory hash is calculated -> All dirs seem to have the same hash
674
                    if (fileInfo is DirectoryInfo)
675
                    {
676
                        //If the directory doesn't exist the Hash property will be empty
677
                        if (String.IsNullOrWhiteSpace(info.Hash))
678
                            //Go on and create the directory
679
                            await client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName, String.Empty, "application/directory");
680
                    }
681
                    else
682
                    {
683

    
684
                        var cloudHash = info.Hash.ToLower();
685

    
686
                        var hash = action.LocalHash.Value;
687
                        var topHash = action.TopHash.Value;
688

    
689
                        //If the file hashes match, abort the upload
690
                        if (hash == cloudHash || topHash == cloudHash)
691
                        {
692
                            //but store any metadata changes 
693
                            StatusKeeper.StoreInfo(fullFileName, info);
694
                            Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
695
                            return;
696
                        }
697

    
698

    
699
                        //Mark the file as modified while we upload it
700
                        StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
701
                        //And then upload it
702

    
703
                        //Upload even small files using the Hashmap. The server may already contain
704
                        //the relevant block
705

    
706
                        //First, calculate the tree hash
707
                        var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize,
708
                                                                              accountInfo.BlockHash, 2);
709

    
710
                        await UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash);
711
                    }
712
                    //If everything succeeds, change the file and overlay status to normal
713
                    StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal);
714
                }
715
                //Notify the Shell to update the overlays
716
                NativeMethods.RaiseChangeNotification(fullFileName);
717
                StatusNotification.NotifyChangedFile(fullFileName);
718
            }
719
            catch (AggregateException ex)
720
            {
721
                var exc = ex.InnerException as WebException;
722
                if (exc == null)
723
                    throw ex.InnerException;
724
                if (HandleUploadWebException(action, exc)) 
725
                    return;
726
                throw;
727
            }
728
            catch (WebException ex)
729
            {
730
                if (HandleUploadWebException(action, ex))
731
                    return;
732
                throw;
733
            }
734
            catch (Exception ex)
735
            {
736
                Log.Error("Unexpected error while uploading file", ex);
737
                throw;
738
            }
739

    
740
        }
741

    
742

    
743

    
744
        private bool HandleUploadWebException(CloudAction action, WebException exc)
745
        {
746
            var response = exc.Response as HttpWebResponse;
747
            if (response == null)
748
                throw exc;
749
            if (response.StatusCode == HttpStatusCode.Unauthorized)
750
            {
751
                Log.Error("Not allowed to upload file", exc);
752
                var message = String.Format("Not allowed to uplad file {0}", action.LocalFile.FullName);
753
                StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Unchanged, FileOverlayStatus.Normal);
754
                StatusNotification.NotifyChange(message, TraceLevel.Warning);
755
                return true;
756
            }
757
            return false;
758
        }
759

    
760
        public async Task UploadWithHashMap(AccountInfo accountInfo,ObjectInfo cloudFile,FileInfo fileInfo,string url,TreeHash treeHash)
761
        {
762
            if (accountInfo == null)
763
                throw new ArgumentNullException("accountInfo");
764
            if (cloudFile==null)
765
                throw new ArgumentNullException("cloudFile");
766
            if (fileInfo == null)
767
                throw new ArgumentNullException("fileInfo");
768
            if (String.IsNullOrWhiteSpace(url))
769
                throw new ArgumentNullException(url);
770
            if (treeHash==null)
771
                throw new ArgumentNullException("treeHash");
772
            if (String.IsNullOrWhiteSpace(cloudFile.Container) )
773
                throw new ArgumentException("Invalid container","cloudFile");
774
            Contract.EndContractBlock();
775

    
776
            var fullFileName = fileInfo.GetProperCapitalization();
777

    
778
            var account = cloudFile.Account ?? accountInfo.UserName;
779
            var container = cloudFile.Container ;
780

    
781
            var client = new CloudFilesClient(accountInfo);
782
            //Send the hashmap to the server            
783
            var missingHashes =  await client.PutHashMap(account, container, url, treeHash);
784
            //If the server returns no missing hashes, we are done
785
            while (missingHashes.Count > 0)
786
            {
787

    
788
                var buffer = new byte[accountInfo.BlockSize];
789
                foreach (var missingHash in missingHashes)
790
                {
791
                    //Find the proper block
792
                    var blockIndex = treeHash.HashDictionary[missingHash];
793
                    var offset = blockIndex*accountInfo.BlockSize;
794

    
795
                    var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);
796

    
797
                    try
798
                    {
799
                        //And upload the block                
800
                        await client.PostBlock(account, container, buffer, 0, read);
801
                        Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex, fullFileName);
802
                    }
803
                    catch (Exception exc)
804
                    {
805
                        Log.ErrorFormat("[ERROR] uploading block {0} of {1}\n{2}", blockIndex, fullFileName, exc);
806
                    }
807

    
808
                }
809

    
810
                //Repeat until there are no more missing hashes                
811
                missingHashes = await client.PutHashMap(account, container, url, treeHash);
812
            }
813
        }
814

    
815

    
816
        public void AddAccount(AccountInfo accountInfo)
817
        {            
818
            if (!_accounts.Contains(accountInfo))
819
                _accounts.Add(accountInfo);
820
        }
821
    }
822

    
823
   
824

    
825

    
826
}