Statistics
| Branch: | Revision:

root / trunk / Pithos.Core / Agents / Downloader.cs @ 8f44fd3a

History | View | Annotate | Download (13.7 kB)

1
´╗┐using System;
2
using System.ComponentModel.Composition;
3
using System.Diagnostics.Contracts;
4
using System.IO;
5
using System.Reflection;
6
using System.Threading.Tasks;
7
using Pithos.Interfaces;
8
using Pithos.Network;
9
using log4net;
10

    
11
namespace Pithos.Core.Agents
12
{
13
    [Export(typeof(Downloader))]
14
    public class Downloader
15
    {
16
        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
17

    
18
        [Import]
19
        private IStatusKeeper StatusKeeper { get; set; }
20

    
21
        
22
        public IStatusNotification StatusNotification { get; set; }
23

    
24
        //Download a file.
25
        public async Task DownloadCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile, string filePath)
26
        {
27
            if (accountInfo == null)
28
                throw new ArgumentNullException("accountInfo");
29
            if (cloudFile == null)
30
                throw new ArgumentNullException("cloudFile");
31
            if (String.IsNullOrWhiteSpace(cloudFile.Account))
32
                throw new ArgumentNullException("cloudFile");
33
            if (String.IsNullOrWhiteSpace(cloudFile.Container))
34
                throw new ArgumentNullException("cloudFile");
35
            if (String.IsNullOrWhiteSpace(filePath))
36
                throw new ArgumentNullException("filePath");
37
            if (!Path.IsPathRooted(filePath))
38
                throw new ArgumentException("The filePath must be rooted", "filePath");
39
            Contract.EndContractBlock();
40

    
41
            using (ThreadContext.Stacks["Operation"].Push("DownloadCloudFile"))
42
            {
43

    
44
                var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath);
45
                var relativeUrl = new Uri(cloudFile.Name, UriKind.Relative);
46

    
47
                var url = relativeUrl.ToString();
48
                if (cloudFile.Name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase))
49
                    return;
50

    
51

    
52
                //Are we already downloading or uploading the file? 
53
                using (var gate = NetworkGate.Acquire(localPath, NetworkOperation.Downloading))
54
                {
55
                    if (gate.Failed)
56
                        return;
57

    
58
                    var client = new CloudFilesClient(accountInfo);
59
                    var account = cloudFile.Account;
60
                    var container = cloudFile.Container;
61

    
62
                    if (cloudFile.IsDirectory)
63
                    {
64
                        if (!Directory.Exists(localPath))
65
                            try
66
                            {
67
                                Directory.CreateDirectory(localPath);
68
                                if (Log.IsDebugEnabled)
69
                                    Log.DebugFormat("Created Directory [{0}]", localPath);
70
                            }
71
                            catch (IOException)
72
                            {
73
                                var localInfo = new FileInfo(localPath);
74
                                if (localInfo.Exists && localInfo.Length == 0)
75
                                {
76
                                    Log.WarnFormat("Malformed directory object detected for [{0}]", localPath);
77
                                    localInfo.Delete();
78
                                    Directory.CreateDirectory(localPath);
79
                                    if (Log.IsDebugEnabled)
80
                                        Log.DebugFormat("Created Directory [{0}]", localPath);
81
                                }
82
                            }
83
                    }
84
                    else
85
                    {
86
                        var isChanged = IsObjectChanged(cloudFile, localPath);
87
                        if (isChanged)
88
                        {
89
                            //Retrieve the hashmap from the server
90
                            var serverHash = await client.GetHashMap(account, container, url);
91
                            //If it's a small file
92
                            if (serverHash.Hashes.Count == 1)
93
                                //Download it in one go
94
                                await
95
                                    DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath);
96
                            //Otherwise download it block by block
97
                            else
98
                                await
99
                                    DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath,
100
                                                       serverHash);
101

    
102
                            if (!cloudFile.IsWritable(accountInfo.UserName))
103
                            {
104
                                var attributes = File.GetAttributes(localPath);
105
                                File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly);
106
                            }
107
                        }
108
                    }
109

    
110
                    //Now we can store the object's metadata without worrying about ghost status entries
111
                    StatusKeeper.StoreInfo(localPath, cloudFile);
112

    
113
                }
114
            }
115
        }
116

    
117
        //Download a file asynchronously using blocks
118
        public async Task DownloadWithBlocks(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath, TreeHash serverHash)
119
        {
120
            if (client == null)
121
                throw new ArgumentNullException("client");
122
            if (cloudFile == null)
123
                throw new ArgumentNullException("cloudFile");
124
            if (relativeUrl == null)
125
                throw new ArgumentNullException("relativeUrl");
126
            if (String.IsNullOrWhiteSpace(filePath))
127
                throw new ArgumentNullException("filePath");
128
            if (!Path.IsPathRooted(filePath))
129
                throw new ArgumentException("The filePath must be rooted", "filePath");
130
            if (serverHash == null)
131
                throw new ArgumentNullException("serverHash");
132
            if (cloudFile.IsDirectory)
133
                throw new ArgumentException("cloudFile is a directory, not a file", "cloudFile");
134
            Contract.EndContractBlock();
135

    
136
            var fileAgent = GetFileAgent(accountInfo);
137
            var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath);
138

    
139
            //Calculate the relative file path for the new file
140
            var relativePath = relativeUrl.RelativeUriToFilePath();
141
            var blockUpdater = new BlockUpdater(fileAgent.CachePath, localPath, relativePath, serverHash);
142

    
143

    
144
            StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing, String.Format("Calculating hashmap for {0} before download", Path.GetFileName(localPath)));
145
            //Calculate the file's treehash
146
            var treeHash = await Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize, serverHash.BlockHash, 2);
147

    
148
            //And compare it with the server's hash
149
            var upHashes = serverHash.GetHashesAsStrings();
150
            var localHashes = treeHash.HashDictionary;
151
            ReportDownloadProgress(Path.GetFileName(localPath), 0, upHashes.Length, cloudFile.Bytes);
152
            for (int i = 0; i < upHashes.Length; i++)
153
            {
154
                //For every non-matching hash
155
                var upHash = upHashes[i];
156
                if (!localHashes.ContainsKey(upHash))
157
                {
158
                    StatusNotification.Notify(new CloudNotification { Data = cloudFile });
159

    
160
                    if (blockUpdater.UseOrphan(i, upHash))
161
                    {
162
                        Log.InfoFormat("[BLOCK GET] ORPHAN FOUND for {0} of {1} for {2}", i, upHashes.Length, localPath);
163
                        continue;
164
                    }
165
                    Log.InfoFormat("[BLOCK GET] START {0} of {1} for {2}", i, upHashes.Length, localPath);
166
                    var start = i * serverHash.BlockSize;
167
                    //To download the last block just pass a null for the end of the range
168
                    long? end = null;
169
                    if (i < upHashes.Length - 1)
170
                        end = ((i + 1) * serverHash.BlockSize);
171

    
172
                    //Download the missing block
173
                    var block = await client.GetBlock(cloudFile.Account, cloudFile.Container, relativeUrl, start, end);
174

    
175
                    //and store it
176
                    blockUpdater.StoreBlock(i, block);
177

    
178

    
179
                    Log.InfoFormat("[BLOCK GET] FINISH {0} of {1} for {2}", i, upHashes.Length, localPath);
180
                }
181
                ReportDownloadProgress(Path.GetFileName(localPath), i, upHashes.Length, cloudFile.Bytes);
182
            }
183

    
184
            //Want to avoid notifications if no changes were made
185
            var hasChanges = blockUpdater.HasBlocks;
186
            blockUpdater.Commit();
187

    
188
            if (hasChanges)
189
                //Notify listeners that a local file has changed
190
                StatusNotification.NotifyChangedFile(localPath);
191

    
192
            Log.InfoFormat("[BLOCK GET] COMPLETE {0}", localPath);
193
        }
194

    
195
        //Download a small file with a single GET operation
196
        private async Task DownloadEntireFileAsync(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath)
197
        {
198
            if (client == null)
199
                throw new ArgumentNullException("client");
200
            if (cloudFile == null)
201
                throw new ArgumentNullException("cloudFile");
202
            if (relativeUrl == null)
203
                throw new ArgumentNullException("relativeUrl");
204
            if (String.IsNullOrWhiteSpace(filePath))
205
                throw new ArgumentNullException("filePath");
206
            if (!Path.IsPathRooted(filePath))
207
                throw new ArgumentException("The localPath must be rooted", "filePath");
208
            if (cloudFile.IsDirectory)
209
                throw new ArgumentException("cloudFile is a directory, not a file", "cloudFile");
210
            Contract.EndContractBlock();
211

    
212
            var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath);
213
            StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing, String.Format("Downloading {0}", Path.GetFileName(localPath)));
214
            StatusNotification.Notify(new CloudNotification { Data = cloudFile });
215

    
216
            var fileAgent = GetFileAgent(accountInfo);
217
            //Calculate the relative file path for the new file
218
            var relativePath = relativeUrl.RelativeUriToFilePath();
219
            //The file will be stored in a temporary location while downloading with an extension .download
220
            var tempPath = Path.Combine(fileAgent.CachePath, relativePath + ".download");
221
            //Make sure the target folder exists. DownloadFileTask will not create the folder
222
            var tempFolder = Path.GetDirectoryName(tempPath);
223
            if (!Directory.Exists(tempFolder))
224
                Directory.CreateDirectory(tempFolder);
225

    
226
            //Download the object to the temporary location
227
            await client.GetObject(cloudFile.Account, cloudFile.Container, relativeUrl.ToString(), tempPath);
228

    
229
            //Create the local folder if it doesn't exist (necessary for shared objects)
230
            var localFolder = Path.GetDirectoryName(localPath);
231
            if (!Directory.Exists(localFolder))
232
                try
233
                {
234
                    Directory.CreateDirectory(localFolder);
235
                }
236
                catch (IOException)
237
                {
238
                    //A file may already exist that has the same name as the new folder.
239
                    //This may be an artifact of the way Pithos handles directories
240
                    var fileInfo = new FileInfo(localFolder);
241
                    if (fileInfo.Exists && fileInfo.Length == 0)
242
                    {
243
                        Log.WarnFormat("Malformed directory object detected for [{0}]", localFolder);
244
                        fileInfo.Delete();
245
                        Directory.CreateDirectory(localFolder);
246
                    }
247
                    else
248
                        throw;
249
                }
250
            //And move it to its actual location once downloading is finished
251
            if (File.Exists(localPath))
252
                File.Replace(tempPath, localPath, null, true);
253
            else
254
                File.Move(tempPath, localPath);
255
            //Notify listeners that a local file has changed
256
            StatusNotification.NotifyChangedFile(localPath);
257

    
258

    
259
        }
260

    
261

    
262
        private void ReportDownloadProgress(string fileName, int block, int totalBlocks, long fileSize)
263
        {
264
            StatusNotification.Notify(totalBlocks == 0
265
                                          ? new ProgressNotification(fileName, "Downloading", 1, 1, fileSize)
266
                                          : new ProgressNotification(fileName, "Downloading", block, totalBlocks, fileSize));
267
        }
268

    
269
        private bool IsObjectChanged(ObjectInfo cloudFile, string localPath)
270
        {
271
            //If the target is a directory, there are no changes to download
272
            if (Directory.Exists(localPath))
273
                return false;
274
            //If the file doesn't exist, we have a chagne
275
            if (!File.Exists(localPath))
276
                return true;
277
            //If there is no stored state, we have a change
278
            var localState = StatusKeeper.GetStateByFilePath(localPath);
279
            if (localState == null)
280
                return true;
281

    
282
            var info = new FileInfo(localPath);
283
            var shortHash = info.ComputeShortHash();
284
            //If the file is different from the stored state, we have a change
285
            if (localState.ShortHash != shortHash)
286
                return true;
287
            //If the top hashes differ, we have a change
288
            return (localState.Checksum != cloudFile.Hash);
289
        }
290

    
291
        private static FileAgent GetFileAgent(AccountInfo accountInfo)
292
        {
293
            return AgentLocator<FileAgent>.Get(accountInfo.AccountPath);
294
        }
295

    
296

    
297
    }
298
}