root / trunk / Pithos.Core / Agents / Downloader.cs @ 8f44fd3a
History | View | Annotate | Download (13.7 kB)
1 | d78d765c | pkanavos | using System; |
---|---|---|---|
2 | d78d765c | pkanavos | using System.ComponentModel.Composition; |
3 | d78d765c | pkanavos | using System.Diagnostics.Contracts; |
4 | d78d765c | pkanavos | using System.IO; |
5 | d78d765c | pkanavos | using System.Reflection; |
6 | d78d765c | pkanavos | using System.Threading.Tasks; |
7 | d78d765c | pkanavos | using Pithos.Interfaces; |
8 | d78d765c | pkanavos | using Pithos.Network; |
9 | d78d765c | pkanavos | using log4net; |
10 | d78d765c | pkanavos | |
11 | d78d765c | pkanavos | namespace Pithos.Core.Agents |
12 | d78d765c | pkanavos | { |
13 | d78d765c | pkanavos | [Export(typeof(Downloader))] |
14 | d78d765c | pkanavos | public class Downloader |
15 | d78d765c | pkanavos | { |
16 | d78d765c | pkanavos | private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); |
17 | d78d765c | pkanavos | |
18 | d78d765c | pkanavos | [Import] |
19 | d78d765c | pkanavos | private IStatusKeeper StatusKeeper { get; set; } |
20 | d78d765c | pkanavos | |
21 | d78d765c | pkanavos | |
22 | d78d765c | pkanavos | public IStatusNotification StatusNotification { get; set; } |
23 | d78d765c | pkanavos | |
24 | d78d765c | pkanavos | //Download a file. |
25 | d78d765c | pkanavos | public async Task DownloadCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile, string filePath) |
26 | d78d765c | pkanavos | { |
27 | d78d765c | pkanavos | if (accountInfo == null) |
28 | d78d765c | pkanavos | throw new ArgumentNullException("accountInfo"); |
29 | d78d765c | pkanavos | if (cloudFile == null) |
30 | d78d765c | pkanavos | throw new ArgumentNullException("cloudFile"); |
31 | d78d765c | pkanavos | if (String.IsNullOrWhiteSpace(cloudFile.Account)) |
32 | d78d765c | pkanavos | throw new ArgumentNullException("cloudFile"); |
33 | d78d765c | pkanavos | if (String.IsNullOrWhiteSpace(cloudFile.Container)) |
34 | d78d765c | pkanavos | throw new ArgumentNullException("cloudFile"); |
35 | d78d765c | pkanavos | if (String.IsNullOrWhiteSpace(filePath)) |
36 | d78d765c | pkanavos | throw new ArgumentNullException("filePath"); |
37 | d78d765c | pkanavos | if (!Path.IsPathRooted(filePath)) |
38 | d78d765c | pkanavos | throw new ArgumentException("The filePath must be rooted", "filePath"); |
39 | d78d765c | pkanavos | Contract.EndContractBlock(); |
40 | d78d765c | pkanavos | |
41 | d78d765c | pkanavos | using (ThreadContext.Stacks["Operation"].Push("DownloadCloudFile")) |
42 | d78d765c | pkanavos | { |
43 | d78d765c | pkanavos | |
44 | d78d765c | pkanavos | var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath); |
45 | d78d765c | pkanavos | var relativeUrl = new Uri(cloudFile.Name, UriKind.Relative); |
46 | d78d765c | pkanavos | |
47 | d78d765c | pkanavos | var url = relativeUrl.ToString(); |
48 | d78d765c | pkanavos | if (cloudFile.Name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase)) |
49 | d78d765c | pkanavos | return; |
50 | d78d765c | pkanavos | |
51 | d78d765c | pkanavos | |
52 | d78d765c | pkanavos | //Are we already downloading or uploading the file? |
53 | d78d765c | pkanavos | using (var gate = NetworkGate.Acquire(localPath, NetworkOperation.Downloading)) |
54 | d78d765c | pkanavos | { |
55 | d78d765c | pkanavos | if (gate.Failed) |
56 | d78d765c | pkanavos | return; |
57 | d78d765c | pkanavos | |
58 | d78d765c | pkanavos | var client = new CloudFilesClient(accountInfo); |
59 | d78d765c | pkanavos | var account = cloudFile.Account; |
60 | d78d765c | pkanavos | var container = cloudFile.Container; |
61 | d78d765c | pkanavos | |
62 | d78d765c | pkanavos | if (cloudFile.IsDirectory) |
63 | d78d765c | pkanavos | { |
64 | d78d765c | pkanavos | if (!Directory.Exists(localPath)) |
65 | d78d765c | pkanavos | try |
66 | d78d765c | pkanavos | { |
67 | d78d765c | pkanavos | Directory.CreateDirectory(localPath); |
68 | d78d765c | pkanavos | if (Log.IsDebugEnabled) |
69 | d78d765c | pkanavos | Log.DebugFormat("Created Directory [{0}]", localPath); |
70 | d78d765c | pkanavos | } |
71 | d78d765c | pkanavos | catch (IOException) |
72 | d78d765c | pkanavos | { |
73 | d78d765c | pkanavos | var localInfo = new FileInfo(localPath); |
74 | d78d765c | pkanavos | if (localInfo.Exists && localInfo.Length == 0) |
75 | d78d765c | pkanavos | { |
76 | d78d765c | pkanavos | Log.WarnFormat("Malformed directory object detected for [{0}]", localPath); |
77 | d78d765c | pkanavos | localInfo.Delete(); |
78 | d78d765c | pkanavos | Directory.CreateDirectory(localPath); |
79 | d78d765c | pkanavos | if (Log.IsDebugEnabled) |
80 | d78d765c | pkanavos | Log.DebugFormat("Created Directory [{0}]", localPath); |
81 | d78d765c | pkanavos | } |
82 | d78d765c | pkanavos | } |
83 | d78d765c | pkanavos | } |
84 | d78d765c | pkanavos | else |
85 | d78d765c | pkanavos | { |
86 | d78d765c | pkanavos | var isChanged = IsObjectChanged(cloudFile, localPath); |
87 | d78d765c | pkanavos | if (isChanged) |
88 | d78d765c | pkanavos | { |
89 | d78d765c | pkanavos | //Retrieve the hashmap from the server |
90 | d78d765c | pkanavos | var serverHash = await client.GetHashMap(account, container, url); |
91 | d78d765c | pkanavos | //If it's a small file |
92 | d78d765c | pkanavos | if (serverHash.Hashes.Count == 1) |
93 | d78d765c | pkanavos | //Download it in one go |
94 | d78d765c | pkanavos | await |
95 | d78d765c | pkanavos | DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath); |
96 | d78d765c | pkanavos | //Otherwise download it block by block |
97 | d78d765c | pkanavos | else |
98 | d78d765c | pkanavos | await |
99 | d78d765c | pkanavos | DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath, |
100 | d78d765c | pkanavos | serverHash); |
101 | d78d765c | pkanavos | |
102 | 8f44fd3a | pkanavos | if (!cloudFile.IsWritable(accountInfo.UserName)) |
103 | d78d765c | pkanavos | { |
104 | d78d765c | pkanavos | var attributes = File.GetAttributes(localPath); |
105 | d78d765c | pkanavos | File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly); |
106 | d78d765c | pkanavos | } |
107 | d78d765c | pkanavos | } |
108 | d78d765c | pkanavos | } |
109 | d78d765c | pkanavos | |
110 | d78d765c | pkanavos | //Now we can store the object's metadata without worrying about ghost status entries |
111 | d78d765c | pkanavos | StatusKeeper.StoreInfo(localPath, cloudFile); |
112 | d78d765c | pkanavos | |
113 | d78d765c | pkanavos | } |
114 | d78d765c | pkanavos | } |
115 | d78d765c | pkanavos | } |
116 | d78d765c | pkanavos | |
117 | d78d765c | pkanavos | //Download a file asynchronously using blocks |
118 | d78d765c | pkanavos | public async Task DownloadWithBlocks(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath, TreeHash serverHash) |
119 | d78d765c | pkanavos | { |
120 | d78d765c | pkanavos | if (client == null) |
121 | d78d765c | pkanavos | throw new ArgumentNullException("client"); |
122 | d78d765c | pkanavos | if (cloudFile == null) |
123 | d78d765c | pkanavos | throw new ArgumentNullException("cloudFile"); |
124 | d78d765c | pkanavos | if (relativeUrl == null) |
125 | d78d765c | pkanavos | throw new ArgumentNullException("relativeUrl"); |
126 | d78d765c | pkanavos | if (String.IsNullOrWhiteSpace(filePath)) |
127 | d78d765c | pkanavos | throw new ArgumentNullException("filePath"); |
128 | d78d765c | pkanavos | if (!Path.IsPathRooted(filePath)) |
129 | d78d765c | pkanavos | throw new ArgumentException("The filePath must be rooted", "filePath"); |
130 | d78d765c | pkanavos | if (serverHash == null) |
131 | d78d765c | pkanavos | throw new ArgumentNullException("serverHash"); |
132 | d78d765c | pkanavos | if (cloudFile.IsDirectory) |
133 | d78d765c | pkanavos | throw new ArgumentException("cloudFile is a directory, not a file", "cloudFile"); |
134 | d78d765c | pkanavos | Contract.EndContractBlock(); |
135 | d78d765c | pkanavos | |
136 | d78d765c | pkanavos | var fileAgent = GetFileAgent(accountInfo); |
137 | d78d765c | pkanavos | var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath); |
138 | d78d765c | pkanavos | |
139 | d78d765c | pkanavos | //Calculate the relative file path for the new file |
140 | d78d765c | pkanavos | var relativePath = relativeUrl.RelativeUriToFilePath(); |
141 | d78d765c | pkanavos | var blockUpdater = new BlockUpdater(fileAgent.CachePath, localPath, relativePath, serverHash); |
142 | d78d765c | pkanavos | |
143 | d78d765c | pkanavos | |
144 | d78d765c | pkanavos | StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing, String.Format("Calculating hashmap for {0} before download", Path.GetFileName(localPath))); |
145 | d78d765c | pkanavos | //Calculate the file's treehash |
146 | d78d765c | pkanavos | var treeHash = await Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize, serverHash.BlockHash, 2); |
147 | d78d765c | pkanavos | |
148 | d78d765c | pkanavos | //And compare it with the server's hash |
149 | d78d765c | pkanavos | var upHashes = serverHash.GetHashesAsStrings(); |
150 | d78d765c | pkanavos | var localHashes = treeHash.HashDictionary; |
151 | d78d765c | pkanavos | ReportDownloadProgress(Path.GetFileName(localPath), 0, upHashes.Length, cloudFile.Bytes); |
152 | d78d765c | pkanavos | for (int i = 0; i < upHashes.Length; i++) |
153 | d78d765c | pkanavos | { |
154 | d78d765c | pkanavos | //For every non-matching hash |
155 | d78d765c | pkanavos | var upHash = upHashes[i]; |
156 | d78d765c | pkanavos | if (!localHashes.ContainsKey(upHash)) |
157 | d78d765c | pkanavos | { |
158 | d78d765c | pkanavos | StatusNotification.Notify(new CloudNotification { Data = cloudFile }); |
159 | d78d765c | pkanavos | |
160 | d78d765c | pkanavos | if (blockUpdater.UseOrphan(i, upHash)) |
161 | d78d765c | pkanavos | { |
162 | d78d765c | pkanavos | Log.InfoFormat("[BLOCK GET] ORPHAN FOUND for {0} of {1} for {2}", i, upHashes.Length, localPath); |
163 | d78d765c | pkanavos | continue; |
164 | d78d765c | pkanavos | } |
165 | d78d765c | pkanavos | Log.InfoFormat("[BLOCK GET] START {0} of {1} for {2}", i, upHashes.Length, localPath); |
166 | d78d765c | pkanavos | var start = i * serverHash.BlockSize; |
167 | d78d765c | pkanavos | //To download the last block just pass a null for the end of the range |
168 | d78d765c | pkanavos | long? end = null; |
169 | d78d765c | pkanavos | if (i < upHashes.Length - 1) |
170 | d78d765c | pkanavos | end = ((i + 1) * serverHash.BlockSize); |
171 | d78d765c | pkanavos | |
172 | d78d765c | pkanavos | //Download the missing block |
173 | d78d765c | pkanavos | var block = await client.GetBlock(cloudFile.Account, cloudFile.Container, relativeUrl, start, end); |
174 | d78d765c | pkanavos | |
175 | d78d765c | pkanavos | //and store it |
176 | d78d765c | pkanavos | blockUpdater.StoreBlock(i, block); |
177 | d78d765c | pkanavos | |
178 | d78d765c | pkanavos | |
179 | d78d765c | pkanavos | Log.InfoFormat("[BLOCK GET] FINISH {0} of {1} for {2}", i, upHashes.Length, localPath); |
180 | d78d765c | pkanavos | } |
181 | d78d765c | pkanavos | ReportDownloadProgress(Path.GetFileName(localPath), i, upHashes.Length, cloudFile.Bytes); |
182 | d78d765c | pkanavos | } |
183 | d78d765c | pkanavos | |
184 | d78d765c | pkanavos | //Want to avoid notifications if no changes were made |
185 | d78d765c | pkanavos | var hasChanges = blockUpdater.HasBlocks; |
186 | d78d765c | pkanavos | blockUpdater.Commit(); |
187 | d78d765c | pkanavos | |
188 | d78d765c | pkanavos | if (hasChanges) |
189 | d78d765c | pkanavos | //Notify listeners that a local file has changed |
190 | d78d765c | pkanavos | StatusNotification.NotifyChangedFile(localPath); |
191 | d78d765c | pkanavos | |
192 | d78d765c | pkanavos | Log.InfoFormat("[BLOCK GET] COMPLETE {0}", localPath); |
193 | d78d765c | pkanavos | } |
194 | d78d765c | pkanavos | |
195 | d78d765c | pkanavos | //Download a small file with a single GET operation |
196 | d78d765c | pkanavos | private async Task DownloadEntireFileAsync(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath) |
197 | d78d765c | pkanavos | { |
198 | d78d765c | pkanavos | if (client == null) |
199 | d78d765c | pkanavos | throw new ArgumentNullException("client"); |
200 | d78d765c | pkanavos | if (cloudFile == null) |
201 | d78d765c | pkanavos | throw new ArgumentNullException("cloudFile"); |
202 | d78d765c | pkanavos | if (relativeUrl == null) |
203 | d78d765c | pkanavos | throw new ArgumentNullException("relativeUrl"); |
204 | d78d765c | pkanavos | if (String.IsNullOrWhiteSpace(filePath)) |
205 | d78d765c | pkanavos | throw new ArgumentNullException("filePath"); |
206 | d78d765c | pkanavos | if (!Path.IsPathRooted(filePath)) |
207 | d78d765c | pkanavos | throw new ArgumentException("The localPath must be rooted", "filePath"); |
208 | d78d765c | pkanavos | if (cloudFile.IsDirectory) |
209 | d78d765c | pkanavos | throw new ArgumentException("cloudFile is a directory, not a file", "cloudFile"); |
210 | d78d765c | pkanavos | Contract.EndContractBlock(); |
211 | d78d765c | pkanavos | |
212 | d78d765c | pkanavos | var localPath = FileInfoExtensions.GetProperFilePathCapitalization(filePath); |
213 | d78d765c | pkanavos | StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing, String.Format("Downloading {0}", Path.GetFileName(localPath))); |
214 | d78d765c | pkanavos | StatusNotification.Notify(new CloudNotification { Data = cloudFile }); |
215 | d78d765c | pkanavos | |
216 | d78d765c | pkanavos | var fileAgent = GetFileAgent(accountInfo); |
217 | d78d765c | pkanavos | //Calculate the relative file path for the new file |
218 | d78d765c | pkanavos | var relativePath = relativeUrl.RelativeUriToFilePath(); |
219 | d78d765c | pkanavos | //The file will be stored in a temporary location while downloading with an extension .download |
220 | d78d765c | pkanavos | var tempPath = Path.Combine(fileAgent.CachePath, relativePath + ".download"); |
221 | d78d765c | pkanavos | //Make sure the target folder exists. DownloadFileTask will not create the folder |
222 | d78d765c | pkanavos | var tempFolder = Path.GetDirectoryName(tempPath); |
223 | d78d765c | pkanavos | if (!Directory.Exists(tempFolder)) |
224 | d78d765c | pkanavos | Directory.CreateDirectory(tempFolder); |
225 | d78d765c | pkanavos | |
226 | d78d765c | pkanavos | //Download the object to the temporary location |
227 | d78d765c | pkanavos | await client.GetObject(cloudFile.Account, cloudFile.Container, relativeUrl.ToString(), tempPath); |
228 | d78d765c | pkanavos | |
229 | d78d765c | pkanavos | //Create the local folder if it doesn't exist (necessary for shared objects) |
230 | d78d765c | pkanavos | var localFolder = Path.GetDirectoryName(localPath); |
231 | d78d765c | pkanavos | if (!Directory.Exists(localFolder)) |
232 | d78d765c | pkanavos | try |
233 | d78d765c | pkanavos | { |
234 | d78d765c | pkanavos | Directory.CreateDirectory(localFolder); |
235 | d78d765c | pkanavos | } |
236 | d78d765c | pkanavos | catch (IOException) |
237 | d78d765c | pkanavos | { |
238 | d78d765c | pkanavos | //A file may already exist that has the same name as the new folder. |
239 | d78d765c | pkanavos | //This may be an artifact of the way Pithos handles directories |
240 | d78d765c | pkanavos | var fileInfo = new FileInfo(localFolder); |
241 | d78d765c | pkanavos | if (fileInfo.Exists && fileInfo.Length == 0) |
242 | d78d765c | pkanavos | { |
243 | d78d765c | pkanavos | Log.WarnFormat("Malformed directory object detected for [{0}]", localFolder); |
244 | d78d765c | pkanavos | fileInfo.Delete(); |
245 | d78d765c | pkanavos | Directory.CreateDirectory(localFolder); |
246 | d78d765c | pkanavos | } |
247 | d78d765c | pkanavos | else |
248 | d78d765c | pkanavos | throw; |
249 | d78d765c | pkanavos | } |
250 | d78d765c | pkanavos | //And move it to its actual location once downloading is finished |
251 | d78d765c | pkanavos | if (File.Exists(localPath)) |
252 | d78d765c | pkanavos | File.Replace(tempPath, localPath, null, true); |
253 | d78d765c | pkanavos | else |
254 | d78d765c | pkanavos | File.Move(tempPath, localPath); |
255 | d78d765c | pkanavos | //Notify listeners that a local file has changed |
256 | d78d765c | pkanavos | StatusNotification.NotifyChangedFile(localPath); |
257 | d78d765c | pkanavos | |
258 | d78d765c | pkanavos | |
259 | d78d765c | pkanavos | } |
260 | d78d765c | pkanavos | |
261 | d78d765c | pkanavos | |
262 | d78d765c | pkanavos | private void ReportDownloadProgress(string fileName, int block, int totalBlocks, long fileSize) |
263 | d78d765c | pkanavos | { |
264 | d78d765c | pkanavos | StatusNotification.Notify(totalBlocks == 0 |
265 | d78d765c | pkanavos | ? new ProgressNotification(fileName, "Downloading", 1, 1, fileSize) |
266 | d78d765c | pkanavos | : new ProgressNotification(fileName, "Downloading", block, totalBlocks, fileSize)); |
267 | d78d765c | pkanavos | } |
268 | d78d765c | pkanavos | |
269 | d78d765c | pkanavos | private bool IsObjectChanged(ObjectInfo cloudFile, string localPath) |
270 | d78d765c | pkanavos | { |
271 | d78d765c | pkanavos | //If the target is a directory, there are no changes to download |
272 | d78d765c | pkanavos | if (Directory.Exists(localPath)) |
273 | d78d765c | pkanavos | return false; |
274 | d78d765c | pkanavos | //If the file doesn't exist, we have a chagne |
275 | d78d765c | pkanavos | if (!File.Exists(localPath)) |
276 | d78d765c | pkanavos | return true; |
277 | d78d765c | pkanavos | //If there is no stored state, we have a change |
278 | d78d765c | pkanavos | var localState = StatusKeeper.GetStateByFilePath(localPath); |
279 | d78d765c | pkanavos | if (localState == null) |
280 | d78d765c | pkanavos | return true; |
281 | d78d765c | pkanavos | |
282 | d78d765c | pkanavos | var info = new FileInfo(localPath); |
283 | d78d765c | pkanavos | var shortHash = info.ComputeShortHash(); |
284 | d78d765c | pkanavos | //If the file is different from the stored state, we have a change |
285 | d78d765c | pkanavos | if (localState.ShortHash != shortHash) |
286 | d78d765c | pkanavos | return true; |
287 | d78d765c | pkanavos | //If the top hashes differ, we have a change |
288 | d78d765c | pkanavos | return (localState.Checksum != cloudFile.Hash); |
289 | d78d765c | pkanavos | } |
290 | d78d765c | pkanavos | |
291 | d78d765c | pkanavos | private static FileAgent GetFileAgent(AccountInfo accountInfo) |
292 | d78d765c | pkanavos | { |
293 | d78d765c | pkanavos | return AgentLocator<FileAgent>.Get(accountInfo.AccountPath); |
294 | d78d765c | pkanavos | } |
295 | d78d765c | pkanavos | |
296 | d78d765c | pkanavos | |
297 | d78d765c | pkanavos | } |
298 | d78d765c | pkanavos | } |