Statistics
| Branch: | Revision:

root / trunk / Pithos.Core / Agents / NetworkAgent.cs @ 5e31048f

History | View | Annotate | Download (41 kB)

1
´╗┐using System;
2
using System.Collections.Generic;
3
using System.ComponentModel;
4
using System.ComponentModel.Composition;
5
using System.Diagnostics;
6
using System.Diagnostics.Contracts;
7
using System.IO;
8
using System.Linq;
9
using System.Net;
10
using System.Text;
11
using System.Threading;
12
using System.Threading.Tasks;
13
using Pithos.Interfaces;
14
using Pithos.Network;
15
using log4net;
16

    
17
namespace Pithos.Core.Agents
18
{
19
    [Export]
20
    public class NetworkAgent
21
    {
22
        private Agent<CloudAction> _agent;
23

    
24

    
25
        [Import]
26
        public IStatusKeeper StatusKeeper { get; set; }
27
        
28
        public IStatusNotification StatusNotification { get; set; }
29
/*
30
        [Import]
31
        public FileAgent FileAgent {get;set;}
32
*/
33

    
34
       /* public int BlockSize { get; set; }
35
        public string BlockHash { get; set; }*/
36

    
37
        private static readonly ILog Log = LogManager.GetLogger("NetworkAgent");
38

    
39
        private List<AccountInfo> _accounts=new List<AccountInfo>();
40

    
41
        public void Start(/*int blockSize, string blockHash*/)
42
        {
43
/*
44
            if (blockSize<0)
45
                throw new ArgumentOutOfRangeException("blockSize");
46
            if (String.IsNullOrWhiteSpace(blockHash))
47
                throw new ArgumentOutOfRangeException("blockHash");
48
            Contract.EndContractBlock();
49
*/
50

    
51
/*
52
            BlockSize = blockSize;
53
            BlockHash = blockHash;
54
*/
55

    
56

    
57
            _agent = Agent<CloudAction>.Start(inbox =>
58
            {
59
                Action loop = null;
60
                loop = () =>
61
                {
62
                    var message = inbox.Receive();
63
                    var process=message.Then(Process,inbox.CancellationToken);
64
                    inbox.LoopAsync(process, loop);
65
                };
66
                loop();
67
            });
68
        }
69

    
70
        private async Task Process(CloudAction action)
71
        {
72
            if (action == null)
73
                throw new ArgumentNullException("action");
74
            if (action.AccountInfo==null)
75
                throw new ArgumentException("The action.AccountInfo is empty","action");
76
            Contract.EndContractBlock();
77

    
78
            var accountInfo = action.AccountInfo;
79

    
80
            using (log4net.ThreadContext.Stacks["NETWORK"].Push("PROCESS"))
81
            {                
82
                Log.InfoFormat("[ACTION] Start Processing {0}", action);
83

    
84
                var localFile = action.LocalFile;
85
                var cloudFile = action.CloudFile;
86
                var downloadPath = action.GetDownloadPath();
87

    
88
                try
89
                {
90

    
91
                    switch (action.Action)
92
                    {
93
                        case CloudActionType.UploadUnconditional:
94
                            await UploadCloudFile(action);
95
                            break;
96
                        case CloudActionType.DownloadUnconditional:
97

    
98
                            await DownloadCloudFile(accountInfo,  cloudFile,downloadPath);
99
                            break;
100
                        case CloudActionType.DeleteCloud:
101
                            DeleteCloudFile(accountInfo, cloudFile);
102
                            break;
103
                        case CloudActionType.RenameCloud:
104
                            var moveAction = (CloudMoveAction)action;
105
                            RenameCloudFile(accountInfo, moveAction);
106
                            break;
107
                        case CloudActionType.MustSynch:
108

    
109
                            if (!File.Exists(downloadPath))
110
                            {                                
111
                                await DownloadCloudFile(accountInfo, cloudFile, downloadPath);
112
                            }
113
                            else
114
                            {
115
                                await SyncFiles(accountInfo, action);
116
                            }
117
                            break;
118
                    }
119
                    Log.InfoFormat("[ACTION] End Processing {0}:{1}->{2}", action.Action, action.LocalFile,
120
                                           action.CloudFile.Name);
121
                }
122
                catch (OperationCanceledException)
123
                {
124
                    throw;
125
                }
126
                catch (FileNotFoundException exc)
127
                {
128
                    Log.ErrorFormat("{0} : {1} -> {2}  failed because the file was not found.\n Rescheduling a delete",
129
                        action.Action, action.LocalFile, action.CloudFile, exc);
130
                    //Post a delete action for the missing file
131
                    Post(new CloudDeleteAction(action));
132
                }
133
                catch (Exception exc)
134
                {
135
                    Log.ErrorFormat("[REQUEUE] {0} : {1} -> {2} due to exception\r\n{3}",
136
                                     action.Action, action.LocalFile, action.CloudFile, exc);
137

    
138
                    _agent.Post(action);
139
                }                
140
            }
141
        }
142

    
143
        private async Task SyncFiles(AccountInfo accountInfo,CloudAction action)
144
        {
145
            if (accountInfo == null)
146
                throw new ArgumentNullException("accountInfo");
147
            if (action==null)
148
                throw new ArgumentNullException("action");
149
            if (action.LocalFile==null)
150
                throw new ArgumentException("The action's local file is not specified","action");
151
            if (!Path.IsPathRooted(action.LocalFile.FullName))
152
                throw new ArgumentException("The action's local file path must be absolute","action");
153
            if (action.CloudFile== null)
154
                throw new ArgumentException("The action's cloud file is not specified", "action");
155
            Contract.EndContractBlock();
156

    
157
            var localFile = action.LocalFile;
158
            var cloudFile = action.CloudFile;
159
            var downloadPath=action.LocalFile.FullName.ToLower();
160

    
161
            var cloudHash = cloudFile.Hash.ToLower();
162
            var localHash = action.LocalHash.Value.ToLower();
163
            var topHash = action.TopHash.Value.ToLower();
164

    
165
            //Not enough to compare only the local hashes, also have to compare the tophashes
166
            
167
            //If any of the hashes match, we are done
168
            if ((cloudHash == localHash || cloudHash == topHash))
169
            {
170
                Log.InfoFormat("Skipping {0}, hashes match",downloadPath);
171
                return;
172
            }
173

    
174
            //The hashes DON'T match. We need to sync
175
            var lastLocalTime = localFile.LastWriteTime;
176
            var lastUpTime = cloudFile.Last_Modified;
177
            
178
            //If the local file is newer upload it
179
            if (lastUpTime <= lastLocalTime)
180
            {
181
                //It probably means it was changed while the app was down                        
182
                UploadCloudFile(action);
183
            }
184
            else
185
            {
186
                //It the cloud file has a later date, it was modified by another user or computer.
187
                //We need to check the local file's status                
188
                var status = StatusKeeper.GetFileStatus(downloadPath);
189
                switch (status)
190
                {
191
                    case FileStatus.Unchanged:                        
192
                        //If the local file's status is Unchanged, we can go on and download the newer cloud file
193
                        await DownloadCloudFile(accountInfo,cloudFile,downloadPath);
194
                        break;
195
                    case FileStatus.Modified:
196
                        //If the local file is Modified, we may have a conflict. In this case we should mark the file as Conflict
197
                        //We can't ensure that a file modified online since the last time will appear as Modified, unless we 
198
                        //index all files before we start listening.                       
199
                    case FileStatus.Created:
200
                        //If the local file is Created, it means that the local and cloud files aren't related,
201
                        // yet they have the same name.
202

    
203
                        //In both cases we must mark the file as in conflict
204
                        ReportConflict(downloadPath);
205
                        break;
206
                    default:
207
                        //Other cases should never occur. Mark them as Conflict as well but log a warning
208
                        ReportConflict(downloadPath);
209
                        Log.WarnFormat("Unexcepted status {0} for file {1}->{2}", status,
210
                                       downloadPath, action.CloudFile.Name);
211
                        break;
212
                }
213
            }
214
        }
215

    
216
        private void ReportConflict(string downloadPath)
217
        {
218
            if (String.IsNullOrWhiteSpace(downloadPath))
219
                throw new ArgumentNullException("downloadPath");
220
            Contract.EndContractBlock();
221

    
222
            StatusKeeper.SetFileOverlayStatus(downloadPath, FileOverlayStatus.Conflict);
223
            var message = String.Format("Conflict detected for file {0}", downloadPath);
224
            Log.Warn(message);
225
            StatusNotification.NotifyChange(message, TraceLevel.Warning);
226
        }
227

    
228
        public void Post(CloudAction cloudAction)
229
        {
230
            if (cloudAction == null)
231
                throw new ArgumentNullException("cloudAction");
232
            if (cloudAction.AccountInfo==null)
233
                throw new ArgumentException("The CloudAction.AccountInfo is empty","cloudAction");
234
            Contract.EndContractBlock();
235
            
236
            //If the action targets a local file, add a treehash calculation
237
            if (cloudAction.LocalFile != null)
238
            {
239
                var accountInfo = cloudAction.AccountInfo;
240
                if (cloudAction.LocalFile.Length>accountInfo.BlockSize)
241
                    cloudAction.TopHash = new Lazy<string>(() => Signature.CalculateTreeHashAsync(cloudAction.LocalFile,
242
                                    accountInfo.BlockSize, accountInfo.BlockHash).Result
243
                                     .TopHash.ToHashString());
244
                else
245
                {
246
                    cloudAction.TopHash=new Lazy<string>(()=> cloudAction.LocalHash.Value);
247
                }
248

    
249
            }
250
            _agent.Post(cloudAction);
251
        }
252

    
253
       /* class ObjectInfoByNameComparer:IEqualityComparer<ObjectInfo>
254
        {
255
            public bool Equals(ObjectInfo x, ObjectInfo y)
256
            {
257
                return x.Name.Equals(y.Name,StringComparison.InvariantCultureIgnoreCase);
258
            }
259

    
260
            public int GetHashCode(ObjectInfo obj)
261
            {
262
                return obj.Name.ToLower().GetHashCode();
263
            }
264
        }*/
265

    
266
        
267

    
268
        //Remote files are polled periodically. Any changes are processed
269
        public Task ProcessRemoteFiles(DateTime? since=null)
270
        {
271
            return Task<Task>.Factory.StartNewDelayed(10000, () =>
272
            {
273
                using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))
274
                {
275
                    //Next time we will check for all changes since the current check minus 1 second
276
                    //This is done to ensure there are no discrepancies due to clock differences
277
                    DateTime nextSince = DateTime.Now.AddSeconds(-1);
278
                    
279
                    var tasks=from accountInfo in _accounts
280
                              select ProcessAccountFiles(accountInfo, since);
281
                    var process=Task.Factory.Iterate(tasks);
282

    
283
                    return process.ContinueWith(t =>
284
                    {
285
                        if (t.IsFaulted)
286
                        {
287
                            Log.Error("Error while processing accounts");
288
                            t.Exception.Handle(exc=>
289
                                                   {
290
                                                       Log.Error("Details:", exc);
291
                                                       return true;
292
                                                   });                            
293
                        }
294
                        ProcessRemoteFiles(nextSince);
295
                    });
296
                }
297
            });            
298
        }
299

    
300
        public Task ProcessAccountFiles(AccountInfo accountInfo,DateTime? since=null)
301
        {   
302
            if (accountInfo==null)
303
                throw new ArgumentNullException("accountInfo");
304
            if (String.IsNullOrWhiteSpace(accountInfo.AccountPath))
305
                throw new ArgumentException("The AccountInfo.AccountPath is empty","accountInfo");
306
            Contract.EndContractBlock();
307

    
308
            using (log4net.ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName))
309
            {
310
                Log.Info("Scheduled");
311
                var client=new CloudFilesClient(accountInfo);
312

    
313
                var containers = client.ListContainers(accountInfo.UserName);
314
                
315
                CreateContainerFolders(accountInfo, containers);
316

    
317

    
318
                //Get the list of server objects changed since the last check
319
                var listObjects = from container in containers
320
                                  select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>
321
                                        client.ListObjects(accountInfo.UserName, container.Name, since),container.Name);
322

    
323
                var listAll = Task.Factory.WhenAll(listObjects.ToArray());
324
                
325
                
326

    
327
                //Get the list of deleted objects since the last check
328
/*
329
                var listTrash = Task<IList<ObjectInfo>>.Factory.StartNew(() =>
330
                                client.ListObjects(accountInfo.UserName, FolderConstants.TrashContainer, since));
331

    
332
                var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(() =>
333
                                client.ListSharedObjects(since));
334

    
335
                var listAll = Task.Factory.TrackedSequence(
336
                    () => listObjects,
337
                    () => listTrash,
338
                    () => listShared);
339
*/
340

    
341

    
342

    
343
                var enqueueFiles = listAll.ContinueWith(task =>
344
                {
345
                    if (task.IsFaulted)
346
                    {
347
                        //ListObjects failed at this point, need to reschedule
348
                        Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {0}", accountInfo.UserName,task.Exception);
349
                        return;
350
                    }
351
                    using (log4net.ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))
352
                    {
353
                        var dict=task.Result.ToDictionary(t=> t.AsyncState);
354
                        
355
                        //Get all non-trash objects. Remember, the container name is stored in AsyncState
356
                        var remoteObjects = from objectList in task.Result
357
                                            where (string)objectList.AsyncState != "trash"
358
                                            from obj in objectList.Result
359
                                            select obj;
360
                                                                       
361
                        var trashObjects = dict["trash"].Result;
362
                        //var sharedObjects = ((Task<IList<ObjectInfo>>) task.Result[2]).Result;
363

    
364
                        //Items with the same name, hash may be both in the container and the trash
365
                        //Don't delete items that exist in the container
366
                        var realTrash = from trash in trashObjects
367
                                        where !remoteObjects.Any(info => info.Hash == trash.Hash)
368
                                        select trash;
369
                        ProcessDeletedFiles(accountInfo,realTrash);                        
370

    
371

    
372
                        var remote = from info in remoteObjects//.Union(sharedObjects)
373
                                     let name = info.Name
374
                                     where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) &&
375
                                           !name.StartsWith(FolderConstants.CacheFolder +"/", StringComparison.InvariantCultureIgnoreCase)
376
                                     select info;
377

    
378
                        //Create a list of actions from the remote files
379
                        var allActions = ObjectsToActions(accountInfo,remote);
380
                       
381
                        //And remove those that are already being processed by the agent
382
                        var distinctActions = allActions
383
                            .Except(_agent.GetEnumerable(), new PithosMonitor.LocalFileComparer())
384
                            .ToList();
385

    
386
                        //Queue all the actions
387
                        foreach (var message in distinctActions)
388
                        {
389
                            Post(message);
390
                        }
391

    
392
                        /*                        //Report the number of new files
393
                                                var remoteCount = distinctActions.Count(action=>
394
                                                    action.Action==CloudActionType.DownloadUnconditional);
395

    
396
                                                if ( remoteCount > 0)
397
                                                    StatusNotification.NotifyChange(String.Format("Processing {0} new files", remoteCount));
398
                        */
399

    
400
                        Log.Info("[LISTENER] End Processing");                        
401
                    }
402
                });
403

    
404
                var log = enqueueFiles.ContinueWith(t =>
405
                {                    
406
                    if (t.IsFaulted)
407
                    {
408
                        Log.Error("[LISTENER] Exception", t.Exception);
409
                    }
410
                    else
411
                    {
412
                        Log.Info("[LISTENER] Finished");
413
                    }
414
                });
415
                return log;
416
            }
417
        }
418

    
419
        private static void CreateContainerFolders(AccountInfo accountInfo, IList<ContainerInfo> containers)
420
        {
421
            var containerPaths = from container in containers
422
                                 let containerPath = Path.Combine(accountInfo.AccountPath, container.Name)
423
                                 where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath)
424
                                 select containerPath;
425

    
426
            foreach (var path in containerPaths)
427
            {
428
                Directory.CreateDirectory(path);
429
            }
430
        }
431

    
432
        //Creates an appropriate action for each server file
433
        private IEnumerable<CloudAction> ObjectsToActions(AccountInfo accountInfo,IEnumerable<ObjectInfo> remote)
434
        {
435
            if (remote==null)
436
                throw new ArgumentNullException();
437
            Contract.EndContractBlock();
438
            var fileAgent = GetFileAgent(accountInfo);
439

    
440
            //In order to avoid multiple iterations over the files, we iterate only once
441
            //over the remote files
442
            foreach (var objectInfo in remote)
443
            {
444
                var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName);
445
                //and remove any matching objects from the list, adding them to the commonObjects list
446
                
447
                if (fileAgent.Exists(relativePath))
448
                {
449
                    var localFile = fileAgent.GetFileInfo(relativePath);
450
                    var state = FileState.FindByFilePath(localFile.FullName);
451
                    //Common files should be checked on a per-case basis to detect differences, which is newer
452

    
453
                    yield return new CloudAction(accountInfo,CloudActionType.MustSynch,
454
                                                   localFile, objectInfo, state, accountInfo.BlockSize,
455
                                                   accountInfo.BlockHash);
456
                }
457
                else
458
                {
459
                    //If there is no match we add them to the localFiles list
460
                    //but only if the file is not marked for deletion
461
                    var targetFile = Path.Combine(accountInfo.AccountPath, relativePath);
462
                    var fileStatus = StatusKeeper.GetFileStatus(targetFile);
463
                    if (fileStatus != FileStatus.Deleted)
464
                    {
465
                        //Remote files should be downloaded
466
                        yield return new CloudDownloadAction(accountInfo,objectInfo);
467
                    }
468
                }
469
            }            
470
        }
471

    
472
        private static FileAgent GetFileAgent(AccountInfo accountInfo)
473
        {
474
            return AgentLocator<FileAgent>.Get(accountInfo.AccountPath);
475
        }
476

    
477
        private void ProcessDeletedFiles(AccountInfo accountInfo,IEnumerable<ObjectInfo> trashObjects)
478
        {
479
            var fileAgent = GetFileAgent(accountInfo);
480
            foreach (var trashObject in trashObjects)
481
            {
482
                var barePath = trashObject.RelativeUrlToFilePath(accountInfo.UserName);
483
                //HACK: Assume only the "pithos" container is used. Must find out what happens when
484
                //deleting a file from a different container
485
                var relativePath = Path.Combine("pithos", barePath);
486
                fileAgent.Delete(relativePath);                                
487
            }
488
        }
489

    
490

    
491
        private void RenameCloudFile(AccountInfo accountInfo,CloudMoveAction action)
492
        {
493
            if (accountInfo==null)
494
                throw new ArgumentNullException("accountInfo");
495
            if (action==null)
496
                throw new ArgumentNullException("action");
497
            if (action.CloudFile==null)
498
                throw new ArgumentException("CloudFile","action");
499
            if (action.LocalFile==null)
500
                throw new ArgumentException("LocalFile","action");
501
            if (action.OldLocalFile==null)
502
                throw new ArgumentException("OldLocalFile","action");
503
            if (action.OldCloudFile==null)
504
                throw new ArgumentException("OldCloudFile","action");
505
            Contract.EndContractBlock();
506
            
507
            var newFilePath = action.LocalFile.FullName;            
508
            //The local file is already renamed
509
            this.StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Modified);
510

    
511

    
512
            var account = action.CloudFile.Account ?? accountInfo.UserName;
513
            var container = action.CloudFile.Container;
514
            
515
            var client = new CloudFilesClient(accountInfo);
516
            client.MoveObject(account, container, action.OldCloudFile.Name, container, action.CloudFile.Name);
517

    
518
            this.StatusKeeper.SetFileStatus(newFilePath, FileStatus.Unchanged);
519
            this.StatusKeeper.SetFileOverlayStatus(newFilePath, FileOverlayStatus.Normal);
520
            NativeMethods.RaiseChangeNotification(newFilePath);
521
        }
522

    
523
        private void DeleteCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile)
524
        {
525
            if (accountInfo == null)
526
                throw new ArgumentNullException("accountInfo");
527
            if (cloudFile==null)
528
                throw new ArgumentNullException("cloudFile");
529

    
530
            if (String.IsNullOrWhiteSpace(cloudFile.Container))
531
                throw new ArgumentException("Invalid container", "cloudFile");
532
            Contract.EndContractBlock();
533
            
534
            var fileAgent = GetFileAgent(accountInfo);
535

    
536
            using ( log4net.ThreadContext.Stacks["DeleteCloudFile"].Push("Delete"))
537
            {
538
                var fileName= cloudFile.RelativeUrlToFilePath(accountInfo.UserName);
539
                var info = fileAgent.GetFileInfo(fileName);                
540
                var fullPath = info.FullName.ToLower();
541

    
542
                this.StatusKeeper.SetFileOverlayStatus(fullPath, FileOverlayStatus.Modified);
543

    
544
                var account = cloudFile.Account ?? accountInfo.UserName;
545
                var container = cloudFile.Container ;//?? FolderConstants.PithosContainer;
546

    
547
                var client = new CloudFilesClient(accountInfo);
548
                client.DeleteObject(account, container, cloudFile.Name);
549

    
550
                this.StatusKeeper.ClearFileStatus(fullPath);
551
            }
552
        }
553

    
554
        //Download a file.
555
        private async Task DownloadCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile , string localPath)
556
        {
557
            if (accountInfo == null)
558
                throw new ArgumentNullException("accountInfo");
559
            if (cloudFile == null)
560
                throw new ArgumentNullException("cloudFile");
561
            if (String.IsNullOrWhiteSpace(cloudFile.Account))
562
                throw new ArgumentNullException("cloudFile");
563
            if (String.IsNullOrWhiteSpace(cloudFile.Container))
564
                throw new ArgumentNullException("cloudFile");
565
            if (String.IsNullOrWhiteSpace(localPath))
566
                throw new ArgumentNullException("localPath");
567
            if (!Path.IsPathRooted(localPath))
568
                throw new ArgumentException("The localPath must be rooted", "localPath");
569
            Contract.EndContractBlock();
570
                       
571
            Uri relativeUrl = new Uri(cloudFile.Name, UriKind.Relative);
572

    
573
            var url = relativeUrl.ToString();
574
            if (cloudFile.Name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase))
575
                return;
576

    
577
            //Are we already downloading or uploading the file? 
578
            using (var gate=NetworkGate.Acquire(localPath, NetworkOperation.Downloading))
579
            {
580
                if (gate.Failed)
581
                    return;
582
                //The file's hashmap will be stored in the same location with the extension .hashmap
583
                //var hashPath = Path.Combine(FileAgent.CachePath, relativePath + ".hashmap");
584
                
585
                var client = new CloudFilesClient(accountInfo);
586
                var account = cloudFile.Account;
587
                var container = cloudFile.Container;
588

    
589
                //Retrieve the hashmap from the server
590
                var serverHash = await client.GetHashMap(account, container, url);
591
                //If it's a small file
592
                if (serverHash.Hashes.Count == 1 )
593
                    //Download it in one go
594
                    await DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath, serverHash);
595
                    //Otherwise download it block by block
596
                else
597
                    await DownloadWithBlocks(accountInfo,client, cloudFile, relativeUrl, localPath, serverHash);                
598

    
599
                if (cloudFile.AllowedTo == "read")
600
                {
601
                    var attributes=File.GetAttributes(localPath);
602
                    File.SetAttributes(localPath,attributes|FileAttributes.ReadOnly);
603
                }
604
                
605
                //Now we can store the object's metadata without worrying about ghost status entries
606
                StatusKeeper.StoreInfo(localPath, cloudFile);
607
                
608
            }
609
        }
610

    
611
        //Download a small file with a single GET operation
612
        private async Task DownloadEntireFileAsync(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string localPath,TreeHash serverHash)
613
        {
614
            if (client == null)
615
                throw new ArgumentNullException("client");
616
            if (cloudFile==null)
617
                throw new ArgumentNullException("cloudFile");
618
            if (relativeUrl == null)
619
                throw new ArgumentNullException("relativeUrl");
620
            if (String.IsNullOrWhiteSpace(localPath))
621
                throw new ArgumentNullException("localPath");
622
            if (!Path.IsPathRooted(localPath))
623
                throw new ArgumentException("The localPath must be rooted", "localPath");
624
            Contract.EndContractBlock();
625

    
626
            //If the file already exists
627
            if (File.Exists(localPath))
628
            {
629
                //First check with MD5 as this is a small file
630
                var localMD5 = Signature.CalculateMD5(localPath);
631
                var cloudHash=serverHash.TopHash.ToHashString();
632
                if (localMD5==cloudHash)
633
                    return;
634
                //Then check with a treehash
635
                var localTreeHash = Signature.CalculateTreeHash(localPath, serverHash.BlockSize, serverHash.BlockHash);
636
                var localHash = localTreeHash.TopHash.ToHashString();
637
                if (localHash==cloudHash)
638
                    return;
639
            }
640

    
641
            var fileAgent = GetFileAgent(accountInfo);
642
            //Calculate the relative file path for the new file
643
            var relativePath = relativeUrl.RelativeUriToFilePath();
644
            //The file will be stored in a temporary location while downloading with an extension .download
645
            var tempPath = Path.Combine(fileAgent.CachePath, relativePath + ".download");
646
            //Make sure the target folder exists. DownloadFileTask will not create the folder
647
            var tempFolder = Path.GetDirectoryName(tempPath);
648
            if (!Directory.Exists(tempFolder))
649
                Directory.CreateDirectory(tempFolder);
650

    
651
            //Download the object to the temporary location
652
            await client.GetObject(cloudFile.Account, cloudFile.Container, relativeUrl.ToString(), tempPath).ContinueWith(t =>
653
            {
654
                t.PropagateExceptions();
655
                //Create the local folder if it doesn't exist (necessary for shared objects)
656
                var localFolder = Path.GetDirectoryName(localPath);
657
                if (!Directory.Exists(localFolder))
658
                    Directory.CreateDirectory(localFolder);
659
                //And move it to its actual location once downloading is finished
660
                if (File.Exists(localPath))
661
                    File.Replace(tempPath,localPath,null,true);
662
                else
663
                    File.Move(tempPath,localPath);
664
                //Notify listeners that a local file has changed
665
                StatusNotification.NotifyChangedFile(localPath);
666

    
667
            });            
668
        }
669

    
670
        //Download a file asynchronously using blocks
671
        public async Task DownloadWithBlocks(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string localPath, TreeHash serverHash)
672
        {
673
            if (client == null)
674
                throw new ArgumentNullException("client");
675
            if (cloudFile == null)
676
                throw new ArgumentNullException("cloudFile");
677
            if (relativeUrl == null)
678
                throw new ArgumentNullException("relativeUrl");
679
            if (String.IsNullOrWhiteSpace(localPath))
680
                throw new ArgumentNullException("localPath");
681
            if (!Path.IsPathRooted(localPath))
682
                throw new ArgumentException("The localPath must be rooted", "localPath");
683
            if (serverHash == null)
684
                throw new ArgumentNullException("serverHash");
685
            Contract.EndContractBlock();
686
            
687
           var fileAgent = GetFileAgent(accountInfo);
688
            
689
            //Calculate the relative file path for the new file
690
            var relativePath = relativeUrl.RelativeUriToFilePath();
691
            var blockUpdater = new BlockUpdater(fileAgent.CachePath, localPath, relativePath, serverHash);
692

    
693
            
694
                        
695
            //Calculate the file's treehash
696
            var treeHash = await Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize, serverHash.BlockHash);
697
                
698
            //And compare it with the server's hash
699
            var upHashes = serverHash.GetHashesAsStrings();
700
            var localHashes = treeHash.HashDictionary;
701
            for (int i = 0; i < upHashes.Length; i++)
702
            {
703
                //For every non-matching hash
704
                var upHash = upHashes[i];
705
                if (!localHashes.ContainsKey(upHash))
706
                {
707
                    if (blockUpdater.UseOrphan(i, upHash))
708
                    {
709
                        Log.InfoFormat("[BLOCK GET] ORPHAN FOUND for {0} of {1} for {2}", i, upHashes.Length, localPath);
710
                        continue;
711
                    }
712
                    Log.InfoFormat("[BLOCK GET] START {0} of {1} for {2}", i, upHashes.Length, localPath);
713
                    var start = i*serverHash.BlockSize;
714
                    //To download the last block just pass a null for the end of the range
715
                    long? end = null;
716
                    if (i < upHashes.Length - 1 )
717
                        end= ((i + 1)*serverHash.BlockSize) ;
718
                            
719
                    //Download the missing block
720
                    var block = await client.GetBlock(cloudFile.Account, cloudFile.Container, relativeUrl, start, end);
721

    
722
                    //and store it
723
                    blockUpdater.StoreBlock(i, block);
724

    
725

    
726
                    Log.InfoFormat("[BLOCK GET] FINISH {0} of {1} for {2}", i, upHashes.Length, localPath);
727
                }
728
            }
729

    
730
            //Want to avoid notifications if no changes were made
731
            var hasChanges = blockUpdater.HasBlocks;
732
            blockUpdater.Commit();
733
            
734
            if (hasChanges)
735
                //Notify listeners that a local file has changed
736
                StatusNotification.NotifyChangedFile(localPath);
737

    
738
            Log.InfoFormat("[BLOCK GET] COMPLETE {0}", localPath);            
739
        }
740

    
741

    
742
        private async Task UploadCloudFile(CloudAction action)
743
        {
744
            if (action == null)
745
                throw new ArgumentNullException("action");           
746
            Contract.EndContractBlock();
747

    
748
            try
749
            {
750
                if (action == null)
751
                    throw new ArgumentNullException("action");
752
                Contract.EndContractBlock();
753

    
754
                var accountInfo = action.AccountInfo;
755

    
756
                var fileInfo = action.LocalFile;
757

    
758
                if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))
759
                    return;
760

    
761
                var relativePath = fileInfo.AsRelativeTo(accountInfo.AccountPath);
762
                if (relativePath.StartsWith(FolderConstants.OthersFolder))
763
                {
764
                    var parts = relativePath.Split('\\');
765
                    var accountName = parts[1];
766
                    var oldName = accountInfo.UserName;
767
                    var absoluteUri = accountInfo.StorageUri.AbsoluteUri;
768
                    var nameIndex = absoluteUri.IndexOf(oldName);
769
                    var root = absoluteUri.Substring(0, nameIndex);
770

    
771
                    accountInfo = new AccountInfo
772
                    {
773
                        UserName = accountName,
774
                        AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
775
                        StorageUri = new Uri(root + accountName),
776
                        BlockHash = accountInfo.BlockHash,
777
                        BlockSize = accountInfo.BlockSize,
778
                        Token = accountInfo.Token
779
                    };
780
                }
781

    
782

    
783
                var fullFileName = fileInfo.FullName;
784
                using (var gate = NetworkGate.Acquire(fullFileName, NetworkOperation.Uploading))
785
                {
786
                    //Abort if the file is already being uploaded or downloaded
787
                    if (gate.Failed)
788
                        return;
789

    
790
                    var cloudFile = action.CloudFile;
791
                    var account = cloudFile.Account ?? accountInfo.UserName;
792

    
793
                    var client = new CloudFilesClient(accountInfo);
794
                    //Even if GetObjectInfo times out, we can proceed with the upload            
795
                    var info = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
796
                    var cloudHash = info.Hash.ToLower();
797

    
798
                    var hash = action.LocalHash.Value;
799
                    var topHash = action.TopHash.Value;
800

    
801
                    //If the file hashes match, abort the upload
802
                    if (hash == cloudHash || topHash == cloudHash)
803
                    {
804
                        //but store any metadata changes 
805
                        this.StatusKeeper.StoreInfo(fullFileName, info);
806
                        Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
807
                        return;
808
                    }
809

    
810
                    if (info.AllowedTo == "read")
811
                        return;
812

    
813
                    //Mark the file as modified while we upload it
814
                    StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
815
                    //And then upload it
816

    
817
                    //Upload even small files using the Hashmap. The server may already containt
818
                    //the relevant folder
819

    
820
                    //First, calculate the tree hash
821
                    var treeHash = await Signature.CalculateTreeHashAsync(fileInfo.FullName, accountInfo.BlockSize,
822
                        accountInfo.BlockHash);
823

    
824
                    await UploadWithHashMap(accountInfo, cloudFile, fileInfo, cloudFile.Name, treeHash);
825

    
826
                    //If everything succeeds, change the file and overlay status to normal
827
                    this.StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal);
828
                }
829
                //Notify the Shell to update the overlays
830
                NativeMethods.RaiseChangeNotification(fullFileName);
831
                StatusNotification.NotifyChangedFile(fullFileName);
832
            }
833
            catch (AggregateException ex)
834
            {
835
                var exc = ex.InnerException as WebException;
836
                if (exc == null)
837
                    throw ex.InnerException;
838
                if (HandleUploadWebException(action, exc)) 
839
                    return;
840
                throw;
841
            }
842
            catch (WebException ex)
843
            {
844
                if (HandleUploadWebException(action, ex))
845
                    return;
846
                throw;
847
            }
848
            catch (Exception ex)
849
            {
850
                Log.Error("Unexpected error while uploading file", ex);
851
                throw;
852
            }
853

    
854
        }
855

    
856
        private bool HandleUploadWebException(CloudAction action, WebException exc)
857
        {
858
            var response = exc.Response as HttpWebResponse;
859
            if (response == null)
860
                throw exc;
861
            if (response.StatusCode == HttpStatusCode.Unauthorized)
862
            {
863
                Log.Error("Not allowed to upload file", exc);
864
                var message = String.Format("Not allowed to uplad file {0}", action.LocalFile.FullName);
865
                StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Unchanged, FileOverlayStatus.Normal);
866
                StatusNotification.NotifyChange(message, TraceLevel.Warning);
867
                return true;
868
            }
869
            return false;
870
        }
871

    
872
        public async Task UploadWithHashMap(AccountInfo accountInfo,ObjectInfo cloudFile,FileInfo fileInfo,string url,TreeHash treeHash)
873
        {
874
            if (accountInfo == null)
875
                throw new ArgumentNullException("accountInfo");
876
            if (cloudFile==null)
877
                throw new ArgumentNullException("cloudFile");
878
            if (fileInfo == null)
879
                throw new ArgumentNullException("fileInfo");
880
            if (String.IsNullOrWhiteSpace(url))
881
                throw new ArgumentNullException(url);
882
            if (treeHash==null)
883
                throw new ArgumentNullException("treeHash");
884
            if (String.IsNullOrWhiteSpace(cloudFile.Container) )
885
                throw new ArgumentException("Invalid container","cloudFile");
886
            Contract.EndContractBlock();
887

    
888
            var fullFileName = fileInfo.FullName;
889

    
890
            var account = cloudFile.Account ?? accountInfo.UserName;
891
            var container = cloudFile.Container ;
892

    
893
            var client = new CloudFilesClient(accountInfo);
894
            //Send the hashmap to the server            
895
            var missingHashes =  await client.PutHashMap(account, container, url, treeHash);
896
            //If the server returns no missing hashes, we are done
897
            while (missingHashes.Count > 0)
898
            {
899

    
900
                var buffer = new byte[accountInfo.BlockSize];
901
                foreach (var missingHash in missingHashes)
902
                {
903
                    //Find the proper block
904
                    var blockIndex = treeHash.HashDictionary[missingHash];
905
                    var offset = blockIndex*accountInfo.BlockSize;
906

    
907
                    var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);
908

    
909
                    try
910
                    {
911
                        //And upload the block                
912
                        await client.PostBlock(account, container, buffer, 0, read);
913
                        Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex, fullFileName);
914
                    }
915
                    catch (Exception exc)
916
                    {
917
                        Log.ErrorFormat("[ERROR] uploading block {0} of {1}\n{2}", blockIndex, fullFileName, exc);
918
                    }
919

    
920
                }
921

    
922
                //Repeat until there are no more missing hashes                
923
                missingHashes = await client.PutHashMap(account, container, url, treeHash);
924
            }
925
        }
926

    
927

    
928
        public void AddAccount(AccountInfo accountInfo)
929
        {            
930
            if (!_accounts.Contains(accountInfo))
931
                _accounts.Add(accountInfo);
932
        }
933
    }
934

    
935
   
936

    
937

    
938
}