root / trunk / Pithos.Core / Agents / BlockUpdater.cs @ 0b346191
History | View | Annotate | Download (9.6 kB)
1 | a64c87c8 | Panagiotis Kanavos | using System; |
---|---|---|---|
2 | a64c87c8 | Panagiotis Kanavos | using System.Collections.Concurrent; |
3 | a64c87c8 | Panagiotis Kanavos | using System.Collections.Generic; |
4 | a64c87c8 | Panagiotis Kanavos | using System.Diagnostics.Contracts; |
5 | a64c87c8 | Panagiotis Kanavos | using System.IO; |
6 | a64c87c8 | Panagiotis Kanavos | using System.Linq; |
7 | 0af3141d | Panagiotis Kanavos | using System.Security.Cryptography; |
8 | a64c87c8 | Panagiotis Kanavos | using System.Text; |
9 | a64c87c8 | Panagiotis Kanavos | using System.Threading.Tasks; |
10 | a64c87c8 | Panagiotis Kanavos | using Pithos.Network; |
11 | a64c87c8 | Panagiotis Kanavos | |
12 | a64c87c8 | Panagiotis Kanavos | namespace Pithos.Core.Agents |
13 | a64c87c8 | Panagiotis Kanavos | { |
14 | a64c87c8 | Panagiotis Kanavos | class BlockUpdater |
15 | a64c87c8 | Panagiotis Kanavos | { |
16 | a64c87c8 | Panagiotis Kanavos | public string FilePath { get; private set; } |
17 | a64c87c8 | Panagiotis Kanavos | public string RelativePath { get; private set; } |
18 | a64c87c8 | Panagiotis Kanavos | |
19 | 77e10b4f | Panagiotis Kanavos | public string CachePath { get; private set; } |
20 | a64c87c8 | Panagiotis Kanavos | |
21 | a64c87c8 | Panagiotis Kanavos | public TreeHash ServerHash { get; private set; } |
22 | a64c87c8 | Panagiotis Kanavos | |
23 | a64c87c8 | Panagiotis Kanavos | public string TempPath { get; private set; } |
24 | 0af3141d | Panagiotis Kanavos | |
25 | 0bd56b7c | Panagiotis Kanavos | public bool HasBlocks |
26 | 0bd56b7c | Panagiotis Kanavos | { |
27 | 0bd56b7c | Panagiotis Kanavos | get { return _blocks.Count>0; } |
28 | 0bd56b7c | Panagiotis Kanavos | } |
29 | 0bd56b7c | Panagiotis Kanavos | |
30 | cfed7823 | Panagiotis Kanavos | readonly ConcurrentDictionary<int, string> _blocks = new ConcurrentDictionary<int, string>(); |
31 | cfed7823 | Panagiotis Kanavos | readonly ConcurrentDictionary<string, string> _orphanBlocks = new ConcurrentDictionary<string, string>(); |
32 | cfed7823 | Panagiotis Kanavos | |
33 | cfed7823 | Panagiotis Kanavos | [ContractInvariantMethod] |
34 | cfed7823 | Panagiotis Kanavos | private void Invariants() |
35 | cfed7823 | Panagiotis Kanavos | { |
36 | 77e10b4f | Panagiotis Kanavos | Contract.Invariant(Path.IsPathRooted(CachePath)); |
37 | cfed7823 | Panagiotis Kanavos | Contract.Invariant(Path.IsPathRooted(FilePath)); |
38 | cfed7823 | Panagiotis Kanavos | Contract.Invariant(Path.IsPathRooted(TempPath)); |
39 | cfed7823 | Panagiotis Kanavos | Contract.Invariant(!Path.IsPathRooted(RelativePath)); |
40 | cfed7823 | Panagiotis Kanavos | Contract.Invariant(_blocks!=null); |
41 | cfed7823 | Panagiotis Kanavos | Contract.Invariant(_orphanBlocks!=null); |
42 | cfed7823 | Panagiotis Kanavos | Contract.Invariant(ServerHash!=null); |
43 | cfed7823 | Panagiotis Kanavos | } |
44 | a64c87c8 | Panagiotis Kanavos | |
45 | 77e10b4f | Panagiotis Kanavos | public BlockUpdater(string cachePath, string filePath, string relativePath,TreeHash serverHash) |
46 | 0af3141d | Panagiotis Kanavos | { |
47 | 77e10b4f | Panagiotis Kanavos | if (String.IsNullOrWhiteSpace(cachePath)) |
48 | 77e10b4f | Panagiotis Kanavos | throw new ArgumentNullException("cachePath"); |
49 | 77e10b4f | Panagiotis Kanavos | if (!Path.IsPathRooted(cachePath)) |
50 | 77e10b4f | Panagiotis Kanavos | throw new ArgumentException("The cachePath must be rooted", "cachePath"); |
51 | 0af3141d | Panagiotis Kanavos | |
52 | 0af3141d | Panagiotis Kanavos | if (string.IsNullOrWhiteSpace(filePath)) |
53 | 0af3141d | Panagiotis Kanavos | throw new ArgumentNullException("filePath"); |
54 | 0af3141d | Panagiotis Kanavos | if (!Path.IsPathRooted(filePath)) |
55 | 0af3141d | Panagiotis Kanavos | throw new ArgumentException("The filePath must be rooted", "filePath"); |
56 | 0af3141d | Panagiotis Kanavos | |
57 | 0af3141d | Panagiotis Kanavos | if (string.IsNullOrWhiteSpace(relativePath)) |
58 | 0af3141d | Panagiotis Kanavos | throw new ArgumentNullException("relativePath"); |
59 | 0af3141d | Panagiotis Kanavos | if (Path.IsPathRooted(relativePath)) |
60 | 0af3141d | Panagiotis Kanavos | throw new ArgumentException("The relativePath must NOT be rooted", "relativePath"); |
61 | 0af3141d | Panagiotis Kanavos | |
62 | 0af3141d | Panagiotis Kanavos | if (serverHash == null) |
63 | 0af3141d | Panagiotis Kanavos | throw new ArgumentNullException("serverHash"); |
64 | 0af3141d | Panagiotis Kanavos | Contract.EndContractBlock(); |
65 | 0af3141d | Panagiotis Kanavos | |
66 | 77e10b4f | Panagiotis Kanavos | CachePath=cachePath; |
67 | a64c87c8 | Panagiotis Kanavos | FilePath = filePath; |
68 | a64c87c8 | Panagiotis Kanavos | RelativePath=relativePath; |
69 | a64c87c8 | Panagiotis Kanavos | ServerHash = serverHash; |
70 | a64c87c8 | Panagiotis Kanavos | //The file will be stored in a temporary location while downloading with an extension .download |
71 | 77e10b4f | Panagiotis Kanavos | TempPath = Path.Combine(CachePath, RelativePath + ".download"); |
72 | cfed7823 | Panagiotis Kanavos | |
73 | cfed7823 | Panagiotis Kanavos | //Need to calculate the directory path because RelativePath may include folders |
74 | cfed7823 | Panagiotis Kanavos | var directoryPath = Path.GetDirectoryName(TempPath); |
75 | cfed7823 | Panagiotis Kanavos | //directoryPath CAN be null if TempPath is a root path |
76 | cfed7823 | Panagiotis Kanavos | if (String.IsNullOrWhiteSpace(directoryPath)) |
77 | cfed7823 | Panagiotis Kanavos | throw new ArgumentException("TempPath"); |
78 | 77e10b4f | Panagiotis Kanavos | //CachePath was absolute so directoryPath is absolute too |
79 | cfed7823 | Panagiotis Kanavos | Contract.Assume(Path.IsPathRooted(directoryPath)); |
80 | cfed7823 | Panagiotis Kanavos | |
81 | a64c87c8 | Panagiotis Kanavos | if (!Directory.Exists(directoryPath)) |
82 | a64c87c8 | Panagiotis Kanavos | Directory.CreateDirectory(directoryPath); |
83 | 0af3141d | Panagiotis Kanavos | |
84 | 0af3141d | Panagiotis Kanavos | LoadOrphans(directoryPath); |
85 | 0af3141d | Panagiotis Kanavos | } |
86 | 0af3141d | Panagiotis Kanavos | |
87 | 0af3141d | Panagiotis Kanavos | private void LoadOrphans(string directoryPath) |
88 | 0af3141d | Panagiotis Kanavos | { |
89 | 0af3141d | Panagiotis Kanavos | if (string.IsNullOrWhiteSpace(directoryPath)) |
90 | 0af3141d | Panagiotis Kanavos | throw new ArgumentNullException("directoryPath"); |
91 | 0af3141d | Panagiotis Kanavos | if (!Path.IsPathRooted(directoryPath)) |
92 | 0af3141d | Panagiotis Kanavos | throw new ArgumentException("The directoryPath must be rooted", "directoryPath"); |
93 | cfed7823 | Panagiotis Kanavos | if (ServerHash==null) |
94 | cfed7823 | Panagiotis Kanavos | throw new InvalidOperationException("ServerHash wasn't initialized"); |
95 | 0af3141d | Panagiotis Kanavos | Contract.EndContractBlock(); |
96 | 0af3141d | Panagiotis Kanavos | |
97 | 0af3141d | Panagiotis Kanavos | var fileNamename = Path.GetFileName(FilePath); |
98 | 0af3141d | Panagiotis Kanavos | var orphans = Directory.GetFiles(directoryPath, fileNamename + ".*"); |
99 | 0af3141d | Panagiotis Kanavos | foreach (var orphan in orphans) |
100 | 0af3141d | Panagiotis Kanavos | { |
101 | 0af3141d | Panagiotis Kanavos | using (HashAlgorithm hasher = HashAlgorithm.Create(ServerHash.BlockHash)) |
102 | 0af3141d | Panagiotis Kanavos | { |
103 | 0af3141d | Panagiotis Kanavos | var buffer=File.ReadAllBytes(orphan); |
104 | 0af3141d | Panagiotis Kanavos | //The server truncates nulls before calculating hashes, have to do the same |
105 | 0af3141d | Panagiotis Kanavos | //Find the last non-null byte, starting from the end |
106 | 0af3141d | Panagiotis Kanavos | var lastByteIndex = Array.FindLastIndex(buffer, buffer.Length-1, aByte => aByte != 0); |
107 | cfed7823 | Panagiotis Kanavos | //lastByteIndex may be -1 if the file was empty. We don't want to use that block file |
108 | cfed7823 | Panagiotis Kanavos | if (lastByteIndex >= 0) |
109 | cfed7823 | Panagiotis Kanavos | { |
110 | cfed7823 | Panagiotis Kanavos | var binHash = hasher.ComputeHash(buffer, 0, lastByteIndex); |
111 | cfed7823 | Panagiotis Kanavos | var hash = binHash.ToHashString(); |
112 | cfed7823 | Panagiotis Kanavos | _orphanBlocks[hash] = orphan; |
113 | cfed7823 | Panagiotis Kanavos | } |
114 | 0af3141d | Panagiotis Kanavos | } |
115 | 0af3141d | Panagiotis Kanavos | } |
116 | a64c87c8 | Panagiotis Kanavos | } |
117 | a64c87c8 | Panagiotis Kanavos | |
118 | a64c87c8 | Panagiotis Kanavos | |
119 | a64c87c8 | Panagiotis Kanavos | public void Commit() |
120 | 0af3141d | Panagiotis Kanavos | { |
121 | 0af3141d | Panagiotis Kanavos | if (String.IsNullOrWhiteSpace(FilePath)) |
122 | 0af3141d | Panagiotis Kanavos | throw new InvalidOperationException("FilePath is empty"); |
123 | 0af3141d | Panagiotis Kanavos | if (String.IsNullOrWhiteSpace(TempPath)) |
124 | 0af3141d | Panagiotis Kanavos | throw new InvalidOperationException("TempPath is empty"); |
125 | 0af3141d | Panagiotis Kanavos | Contract.EndContractBlock(); |
126 | 0af3141d | Panagiotis Kanavos | |
127 | a64c87c8 | Panagiotis Kanavos | //Copy the file to a temporary location. Changes will be made to the |
128 | a64c87c8 | Panagiotis Kanavos | //temporary file, then it will replace the original file |
129 | 0af3141d | Panagiotis Kanavos | if (File.Exists(FilePath)) |
130 | 0af3141d | Panagiotis Kanavos | File.Copy(FilePath, TempPath, true); |
131 | a64c87c8 | Panagiotis Kanavos | |
132 | a64c87c8 | Panagiotis Kanavos | //Set the size of the file to the size specified in the treehash |
133 | cfed7823 | Panagiotis Kanavos | //This will also create an empty file if the file doesn't exist |
134 | cfed7823 | Panagiotis Kanavos | |
135 | cfed7823 | Panagiotis Kanavos | |
136 | a64c87c8 | Panagiotis Kanavos | SetFileSize(TempPath, ServerHash.Bytes); |
137 | a64c87c8 | Panagiotis Kanavos | |
138 | a64c87c8 | Panagiotis Kanavos | //Update the temporary file with the data from the blocks |
139 | a64c87c8 | Panagiotis Kanavos | using (var stream = File.OpenWrite(TempPath)) |
140 | a64c87c8 | Panagiotis Kanavos | { |
141 | a64c87c8 | Panagiotis Kanavos | foreach (var block in _blocks) |
142 | a64c87c8 | Panagiotis Kanavos | { |
143 | a64c87c8 | Panagiotis Kanavos | var blockPath = block.Value; |
144 | a64c87c8 | Panagiotis Kanavos | var blockIndex = block.Key; |
145 | a64c87c8 | Panagiotis Kanavos | using (var blockStream = File.OpenRead(blockPath)) |
146 | a64c87c8 | Panagiotis Kanavos | { |
147 | a64c87c8 | Panagiotis Kanavos | var offset = blockIndex*ServerHash.BlockSize; |
148 | a64c87c8 | Panagiotis Kanavos | stream.Seek(offset, SeekOrigin.Begin); |
149 | a64c87c8 | Panagiotis Kanavos | blockStream.CopyTo(stream); |
150 | a64c87c8 | Panagiotis Kanavos | } |
151 | a64c87c8 | Panagiotis Kanavos | } |
152 | a64c87c8 | Panagiotis Kanavos | } |
153 | a64c87c8 | Panagiotis Kanavos | SwapFiles(); |
154 | a64c87c8 | Panagiotis Kanavos | |
155 | a64c87c8 | Panagiotis Kanavos | ClearBlocks(); |
156 | a64c87c8 | Panagiotis Kanavos | } |
157 | a64c87c8 | Panagiotis Kanavos | |
158 | a64c87c8 | Panagiotis Kanavos | private void SwapFiles() |
159 | a64c87c8 | Panagiotis Kanavos | { |
160 | 0af3141d | Panagiotis Kanavos | if (String.IsNullOrWhiteSpace(FilePath)) |
161 | 0af3141d | Panagiotis Kanavos | throw new InvalidOperationException("FilePath is empty"); |
162 | 0af3141d | Panagiotis Kanavos | if (String.IsNullOrWhiteSpace(TempPath)) |
163 | 0af3141d | Panagiotis Kanavos | throw new InvalidOperationException("TempPath is empty"); |
164 | 0af3141d | Panagiotis Kanavos | Contract.EndContractBlock(); |
165 | 0af3141d | Panagiotis Kanavos | |
166 | a64c87c8 | Panagiotis Kanavos | if (File.Exists(FilePath)) |
167 | a64c87c8 | Panagiotis Kanavos | File.Replace(TempPath, FilePath, null, true); |
168 | a64c87c8 | Panagiotis Kanavos | else |
169 | a64c87c8 | Panagiotis Kanavos | File.Move(TempPath, FilePath); |
170 | a64c87c8 | Panagiotis Kanavos | } |
171 | a64c87c8 | Panagiotis Kanavos | |
172 | a64c87c8 | Panagiotis Kanavos | private void ClearBlocks() |
173 | a64c87c8 | Panagiotis Kanavos | { |
174 | 0af3141d | Panagiotis Kanavos | //Get all the the block paths, orphan or not |
175 | 0af3141d | Panagiotis Kanavos | var paths= _blocks.Select(pair => pair.Value) |
176 | 0af3141d | Panagiotis Kanavos | .Union(_orphanBlocks.Select(pair => pair.Value)); |
177 | 0af3141d | Panagiotis Kanavos | foreach (var filePath in paths) |
178 | a64c87c8 | Panagiotis Kanavos | { |
179 | a64c87c8 | Panagiotis Kanavos | File.Delete(filePath); |
180 | a64c87c8 | Panagiotis Kanavos | } |
181 | 0af3141d | Panagiotis Kanavos | |
182 | a64c87c8 | Panagiotis Kanavos | File.Delete(TempPath); |
183 | a64c87c8 | Panagiotis Kanavos | _blocks.Clear(); |
184 | 0af3141d | Panagiotis Kanavos | _orphanBlocks.Clear(); |
185 | a64c87c8 | Panagiotis Kanavos | } |
186 | a64c87c8 | Panagiotis Kanavos | |
187 | a64c87c8 | Panagiotis Kanavos | //Change the file's size, possibly truncating or adding to it |
188 | a64c87c8 | Panagiotis Kanavos | private void SetFileSize(string filePath, long fileSize) |
189 | a64c87c8 | Panagiotis Kanavos | { |
190 | a64c87c8 | Panagiotis Kanavos | if (String.IsNullOrWhiteSpace(filePath)) |
191 | a64c87c8 | Panagiotis Kanavos | throw new ArgumentNullException("filePath"); |
192 | a64c87c8 | Panagiotis Kanavos | if (!Path.IsPathRooted(filePath)) |
193 | a64c87c8 | Panagiotis Kanavos | throw new ArgumentException("The filePath must be rooted", "filePath"); |
194 | a64c87c8 | Panagiotis Kanavos | if (fileSize < 0) |
195 | a64c87c8 | Panagiotis Kanavos | throw new ArgumentOutOfRangeException("fileSize"); |
196 | a64c87c8 | Panagiotis Kanavos | Contract.EndContractBlock(); |
197 | a64c87c8 | Panagiotis Kanavos | |
198 | a64c87c8 | Panagiotis Kanavos | using (var stream = File.Open(filePath, FileMode.OpenOrCreate, FileAccess.Write)) |
199 | a64c87c8 | Panagiotis Kanavos | { |
200 | a64c87c8 | Panagiotis Kanavos | stream.SetLength(fileSize); |
201 | a64c87c8 | Panagiotis Kanavos | } |
202 | a64c87c8 | Panagiotis Kanavos | } |
203 | a64c87c8 | Panagiotis Kanavos | |
204 | a64c87c8 | Panagiotis Kanavos | /* //Check whether we should copy the local file to a temp path |
205 | a64c87c8 | Panagiotis Kanavos | private bool ShouldCopy(string localPath, string tempPath) |
206 | a64c87c8 | Panagiotis Kanavos | { |
207 | a64c87c8 | Panagiotis Kanavos | //No need to copy if there is no file |
208 | a64c87c8 | Panagiotis Kanavos | if (!File.Exists(localPath)) |
209 | a64c87c8 | Panagiotis Kanavos | return false; |
210 | a64c87c8 | Panagiotis Kanavos | |
211 | a64c87c8 | Panagiotis Kanavos | //If there is no temp file, go ahead and copy |
212 | a64c87c8 | Panagiotis Kanavos | if (!File.Exists(tempPath)) |
213 | a64c87c8 | Panagiotis Kanavos | return true; |
214 | a64c87c8 | Panagiotis Kanavos | |
215 | a64c87c8 | Panagiotis Kanavos | //If there is a temp file and is newer than the actual file, don't copy |
216 | a64c87c8 | Panagiotis Kanavos | var localLastWrite = File.GetLastWriteTime(localPath); |
217 | a64c87c8 | Panagiotis Kanavos | var tempLastWrite = File.GetLastWriteTime(tempPath); |
218 | a64c87c8 | Panagiotis Kanavos | |
219 | a64c87c8 | Panagiotis Kanavos | //This could mean there is an interrupted download in progress |
220 | a64c87c8 | Panagiotis Kanavos | return (tempLastWrite < localLastWrite); |
221 | a64c87c8 | Panagiotis Kanavos | }*/ |
222 | a64c87c8 | Panagiotis Kanavos | |
223 | 0af3141d | Panagiotis Kanavos | |
224 | 0af3141d | Panagiotis Kanavos | public bool UseOrphan(int blockIndex, string blockHash) |
225 | 0af3141d | Panagiotis Kanavos | { |
226 | 0af3141d | Panagiotis Kanavos | string blockPath=null; |
227 | 0af3141d | Panagiotis Kanavos | if (_orphanBlocks.TryGetValue(blockHash,out blockPath)) |
228 | 0af3141d | Panagiotis Kanavos | { |
229 | 0af3141d | Panagiotis Kanavos | _blocks[blockIndex] = blockPath; |
230 | 0af3141d | Panagiotis Kanavos | return true; |
231 | 0af3141d | Panagiotis Kanavos | } |
232 | 0af3141d | Panagiotis Kanavos | return false; |
233 | 0af3141d | Panagiotis Kanavos | } |
234 | a64c87c8 | Panagiotis Kanavos | |
235 | a64c87c8 | Panagiotis Kanavos | public Task StoreBlock(int blockIndex,byte[] buffer) |
236 | a64c87c8 | Panagiotis Kanavos | { |
237 | 0af3141d | Panagiotis Kanavos | var blockPath = String.Format("{0}.{1:000000}", TempPath, blockIndex); |
238 | a64c87c8 | Panagiotis Kanavos | _blocks[blockIndex] = blockPath; |
239 | 0af3141d | Panagiotis Kanavos | //Remove any orphan files |
240 | 0af3141d | Panagiotis Kanavos | if (File.Exists(blockPath)) |
241 | 0af3141d | Panagiotis Kanavos | File.Delete(blockPath); |
242 | 0af3141d | Panagiotis Kanavos | |
243 | a64c87c8 | Panagiotis Kanavos | return FileAsync.WriteAllBytes(blockPath, buffer); |
244 | a64c87c8 | Panagiotis Kanavos | } |
245 | a64c87c8 | Panagiotis Kanavos | |
246 | 73cdd135 | Panagiotis Kanavos | |
247 | a64c87c8 | Panagiotis Kanavos | |
248 | a64c87c8 | Panagiotis Kanavos | } |
249 | a64c87c8 | Panagiotis Kanavos | } |