root / trunk / Pithos.Core / Agents / Downloader.cs @ 8f44fd3a
History | View | Annotate | Download (13.7 kB)
1 |
using System; |
---|---|
2 |
using System.ComponentModel.Composition; |
3 |
using System.Diagnostics.Contracts; |
4 |
using System.IO; |
5 |
using System.Reflection; |
6 |
using System.Threading.Tasks; |
7 |
using Pithos.Interfaces; |
8 |
using Pithos.Network; |
9 |
using log4net; |
10 |
|
11 |
namespace Pithos.Core.Agents |
12 |
{ |
13 |
[Export(typeof(Downloader))] |
14 |
public class Downloader |
15 |
{ |
16 |
private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); |
17 |
|
18 |
[Import] |
19 |
private IStatusKeeper StatusKeeper { get; set; } |
20 |
|
21 |
|
22 |
public IStatusNotification StatusNotification { get; set; } |
23 |
|
24 |
//Download a file. |
25 |
public async Task DownloadCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile, string filePath) |
26 |
{ |
27 |
if (accountInfo == null) |
28 |
throw new ArgumentNullException("accountInfo"); |
29 |
if (cloudFile == null) |
30 |
throw new ArgumentNullException("cloudFile"); |
31 |
if (String.IsNullOrWhiteSpace(cloudFile.Account)) |
32 |
throw new ArgumentNullException("cloudFile"); |
33 |
if (String.IsNullOrWhiteSpace(cloudFile.Container)) |
34 |
throw new ArgumentNullException("cloudFile"); |
35 |
if (String.IsNullOrWhiteSpace(filePath)) |
36 |
throw new ArgumentNullException("filePath"); |
37 |
if (!Path.IsPathRooted(filePath)) |
38 |
throw new ArgumentException("The filePath must be rooted", "filePath"); |
39 |
Contract.EndContractBlock(); |
40 |
|
41 |
using (ThreadContext.Stacks["Operation"].Push("DownloadCloudFile")) |
42 |
{ |
43 |
|
44 |
var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath); |
45 |
var relativeUrl = new Uri(cloudFile.Name, UriKind.Relative); |
46 |
|
47 |
var url = relativeUrl.ToString(); |
48 |
if (cloudFile.Name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase)) |
49 |
return; |
50 |
|
51 |
|
52 |
//Are we already downloading or uploading the file? |
53 |
using (var gate = NetworkGate.Acquire(localPath, NetworkOperation.Downloading)) |
54 |
{ |
55 |
if (gate.Failed) |
56 |
return; |
57 |
|
58 |
var client = new CloudFilesClient(accountInfo); |
59 |
var account = cloudFile.Account; |
60 |
var container = cloudFile.Container; |
61 |
|
62 |
if (cloudFile.IsDirectory) |
63 |
{ |
64 |
if (!Directory.Exists(localPath)) |
65 |
try |
66 |
{ |
67 |
Directory.CreateDirectory(localPath); |
68 |
if (Log.IsDebugEnabled) |
69 |
Log.DebugFormat("Created Directory [{0}]", localPath); |
70 |
} |
71 |
catch (IOException) |
72 |
{ |
73 |
var localInfo = new FileInfo(localPath); |
74 |
if (localInfo.Exists && localInfo.Length == 0) |
75 |
{ |
76 |
Log.WarnFormat("Malformed directory object detected for [{0}]", localPath); |
77 |
localInfo.Delete(); |
78 |
Directory.CreateDirectory(localPath); |
79 |
if (Log.IsDebugEnabled) |
80 |
Log.DebugFormat("Created Directory [{0}]", localPath); |
81 |
} |
82 |
} |
83 |
} |
84 |
else |
85 |
{ |
86 |
var isChanged = IsObjectChanged(cloudFile, localPath); |
87 |
if (isChanged) |
88 |
{ |
89 |
//Retrieve the hashmap from the server |
90 |
var serverHash = await client.GetHashMap(account, container, url); |
91 |
//If it's a small file |
92 |
if (serverHash.Hashes.Count == 1) |
93 |
//Download it in one go |
94 |
await |
95 |
DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath); |
96 |
//Otherwise download it block by block |
97 |
else |
98 |
await |
99 |
DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath, |
100 |
serverHash); |
101 |
|
102 |
if (!cloudFile.IsWritable(accountInfo.UserName)) |
103 |
{ |
104 |
var attributes = File.GetAttributes(localPath); |
105 |
File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly); |
106 |
} |
107 |
} |
108 |
} |
109 |
|
110 |
//Now we can store the object's metadata without worrying about ghost status entries |
111 |
StatusKeeper.StoreInfo(localPath, cloudFile); |
112 |
|
113 |
} |
114 |
} |
115 |
} |
116 |
|
117 |
//Download a file asynchronously using blocks |
118 |
public async Task DownloadWithBlocks(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath, TreeHash serverHash) |
119 |
{ |
120 |
if (client == null) |
121 |
throw new ArgumentNullException("client"); |
122 |
if (cloudFile == null) |
123 |
throw new ArgumentNullException("cloudFile"); |
124 |
if (relativeUrl == null) |
125 |
throw new ArgumentNullException("relativeUrl"); |
126 |
if (String.IsNullOrWhiteSpace(filePath)) |
127 |
throw new ArgumentNullException("filePath"); |
128 |
if (!Path.IsPathRooted(filePath)) |
129 |
throw new ArgumentException("The filePath must be rooted", "filePath"); |
130 |
if (serverHash == null) |
131 |
throw new ArgumentNullException("serverHash"); |
132 |
if (cloudFile.IsDirectory) |
133 |
throw new ArgumentException("cloudFile is a directory, not a file", "cloudFile"); |
134 |
Contract.EndContractBlock(); |
135 |
|
136 |
var fileAgent = GetFileAgent(accountInfo); |
137 |
var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath); |
138 |
|
139 |
//Calculate the relative file path for the new file |
140 |
var relativePath = relativeUrl.RelativeUriToFilePath(); |
141 |
var blockUpdater = new BlockUpdater(fileAgent.CachePath, localPath, relativePath, serverHash); |
142 |
|
143 |
|
144 |
StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing, String.Format("Calculating hashmap for {0} before download", Path.GetFileName(localPath))); |
145 |
//Calculate the file's treehash |
146 |
var treeHash = await Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize, serverHash.BlockHash, 2); |
147 |
|
148 |
//And compare it with the server's hash |
149 |
var upHashes = serverHash.GetHashesAsStrings(); |
150 |
var localHashes = treeHash.HashDictionary; |
151 |
ReportDownloadProgress(Path.GetFileName(localPath), 0, upHashes.Length, cloudFile.Bytes); |
152 |
for (int i = 0; i < upHashes.Length; i++) |
153 |
{ |
154 |
//For every non-matching hash |
155 |
var upHash = upHashes[i]; |
156 |
if (!localHashes.ContainsKey(upHash)) |
157 |
{ |
158 |
StatusNotification.Notify(new CloudNotification { Data = cloudFile }); |
159 |
|
160 |
if (blockUpdater.UseOrphan(i, upHash)) |
161 |
{ |
162 |
Log.InfoFormat("[BLOCK GET] ORPHAN FOUND for {0} of {1} for {2}", i, upHashes.Length, localPath); |
163 |
continue; |
164 |
} |
165 |
Log.InfoFormat("[BLOCK GET] START {0} of {1} for {2}", i, upHashes.Length, localPath); |
166 |
var start = i * serverHash.BlockSize; |
167 |
//To download the last block just pass a null for the end of the range |
168 |
long? end = null; |
169 |
if (i < upHashes.Length - 1) |
170 |
end = ((i + 1) * serverHash.BlockSize); |
171 |
|
172 |
//Download the missing block |
173 |
var block = await client.GetBlock(cloudFile.Account, cloudFile.Container, relativeUrl, start, end); |
174 |
|
175 |
//and store it |
176 |
blockUpdater.StoreBlock(i, block); |
177 |
|
178 |
|
179 |
Log.InfoFormat("[BLOCK GET] FINISH {0} of {1} for {2}", i, upHashes.Length, localPath); |
180 |
} |
181 |
ReportDownloadProgress(Path.GetFileName(localPath), i, upHashes.Length, cloudFile.Bytes); |
182 |
} |
183 |
|
184 |
//Want to avoid notifications if no changes were made |
185 |
var hasChanges = blockUpdater.HasBlocks; |
186 |
blockUpdater.Commit(); |
187 |
|
188 |
if (hasChanges) |
189 |
//Notify listeners that a local file has changed |
190 |
StatusNotification.NotifyChangedFile(localPath); |
191 |
|
192 |
Log.InfoFormat("[BLOCK GET] COMPLETE {0}", localPath); |
193 |
} |
194 |
|
195 |
//Download a small file with a single GET operation |
196 |
private async Task DownloadEntireFileAsync(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath) |
197 |
{ |
198 |
if (client == null) |
199 |
throw new ArgumentNullException("client"); |
200 |
if (cloudFile == null) |
201 |
throw new ArgumentNullException("cloudFile"); |
202 |
if (relativeUrl == null) |
203 |
throw new ArgumentNullException("relativeUrl"); |
204 |
if (String.IsNullOrWhiteSpace(filePath)) |
205 |
throw new ArgumentNullException("filePath"); |
206 |
if (!Path.IsPathRooted(filePath)) |
207 |
throw new ArgumentException("The localPath must be rooted", "filePath"); |
208 |
if (cloudFile.IsDirectory) |
209 |
throw new ArgumentException("cloudFile is a directory, not a file", "cloudFile"); |
210 |
Contract.EndContractBlock(); |
211 |
|
212 |
var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath); |
213 |
StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing, String.Format("Downloading {0}", Path.GetFileName(localPath))); |
214 |
StatusNotification.Notify(new CloudNotification { Data = cloudFile }); |
215 |
|
216 |
var fileAgent = GetFileAgent(accountInfo); |
217 |
//Calculate the relative file path for the new file |
218 |
var relativePath = relativeUrl.RelativeUriToFilePath(); |
219 |
//The file will be stored in a temporary location while downloading with an extension .download |
220 |
var tempPath = Path.Combine(fileAgent.CachePath, relativePath + ".download"); |
221 |
//Make sure the target folder exists. DownloadFileTask will not create the folder |
222 |
var tempFolder = Path.GetDirectoryName(tempPath); |
223 |
if (!Directory.Exists(tempFolder)) |
224 |
Directory.CreateDirectory(tempFolder); |
225 |
|
226 |
//Download the object to the temporary location |
227 |
await client.GetObject(cloudFile.Account, cloudFile.Container, relativeUrl.ToString(), tempPath); |
228 |
|
229 |
//Create the local folder if it doesn't exist (necessary for shared objects) |
230 |
var localFolder = Path.GetDirectoryName(localPath); |
231 |
if (!Directory.Exists(localFolder)) |
232 |
try |
233 |
{ |
234 |
Directory.CreateDirectory(localFolder); |
235 |
} |
236 |
catch (IOException) |
237 |
{ |
238 |
//A file may already exist that has the same name as the new folder. |
239 |
//This may be an artifact of the way Pithos handles directories |
240 |
var fileInfo = new FileInfo(localFolder); |
241 |
if (fileInfo.Exists && fileInfo.Length == 0) |
242 |
{ |
243 |
Log.WarnFormat("Malformed directory object detected for [{0}]", localFolder); |
244 |
fileInfo.Delete(); |
245 |
Directory.CreateDirectory(localFolder); |
246 |
} |
247 |
else |
248 |
throw; |
249 |
} |
250 |
//And move it to its actual location once downloading is finished |
251 |
if (File.Exists(localPath)) |
252 |
File.Replace(tempPath, localPath, null, true); |
253 |
else |
254 |
File.Move(tempPath, localPath); |
255 |
//Notify listeners that a local file has changed |
256 |
StatusNotification.NotifyChangedFile(localPath); |
257 |
|
258 |
|
259 |
} |
260 |
|
261 |
|
262 |
private void ReportDownloadProgress(string fileName, int block, int totalBlocks, long fileSize) |
263 |
{ |
264 |
StatusNotification.Notify(totalBlocks == 0 |
265 |
? new ProgressNotification(fileName, "Downloading", 1, 1, fileSize) |
266 |
: new ProgressNotification(fileName, "Downloading", block, totalBlocks, fileSize)); |
267 |
} |
268 |
|
269 |
private bool IsObjectChanged(ObjectInfo cloudFile, string localPath) |
270 |
{ |
271 |
//If the target is a directory, there are no changes to download |
272 |
if (Directory.Exists(localPath)) |
273 |
return false; |
274 |
//If the file doesn't exist, we have a chagne |
275 |
if (!File.Exists(localPath)) |
276 |
return true; |
277 |
//If there is no stored state, we have a change |
278 |
var localState = StatusKeeper.GetStateByFilePath(localPath); |
279 |
if (localState == null) |
280 |
return true; |
281 |
|
282 |
var info = new FileInfo(localPath); |
283 |
var shortHash = info.ComputeShortHash(); |
284 |
//If the file is different from the stored state, we have a change |
285 |
if (localState.ShortHash != shortHash) |
286 |
return true; |
287 |
//If the top hashes differ, we have a change |
288 |
return (localState.Checksum != cloudFile.Hash); |
289 |
} |
290 |
|
291 |
private static FileAgent GetFileAgent(AccountInfo accountInfo) |
292 |
{ |
293 |
return AgentLocator<FileAgent>.Get(accountInfo.AccountPath); |
294 |
} |
295 |
|
296 |
|
297 |
} |
298 |
} |