code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/BlockUpdater.cs

   1 #region\r
   2 /* -----------------------------------------------------------------------\r
   3  * <copyright file="BlockUpdater.cs" company="GRNet">\r
   4  * \r
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.\r
   6  *\r
   7  * Redistribution and use in source and binary forms, with or\r
   8  * without modification, are permitted provided that the following\r
   9  * conditions are met:\r
  10  *\r
  11  *   1. Redistributions of source code must retain the above\r
  12  *      copyright notice, this list of conditions and the following\r
  13  *      disclaimer.\r
  14  *\r
  15  *   2. Redistributions in binary form must reproduce the above\r
  16  *      copyright notice, this list of conditions and the following\r
  17  *      disclaimer in the documentation and/or other materials\r
  18  *      provided with the distribution.\r
  19  *\r
  20  *\r
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS\r
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\r
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\r
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR\r
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\r
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\r
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF\r
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED\r
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\r
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN\r
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE\r
  32  * POSSIBILITY OF SUCH DAMAGE.\r
  33  *\r
  34  * The views and conclusions contained in the software and\r
  35  * documentation are those of the authors and should not be\r
  36  * interpreted as representing official policies, either expressed\r
  37  * or implied, of GRNET S.A.\r
  38  * </copyright>\r
  39  * -----------------------------------------------------------------------\r
  40  */\r
  41 #endregion\r
  42 using System;\r
  43 using System.Collections.Concurrent;\r
  44 using System.Diagnostics.Contracts;\r
  45 using System.IO;\r
  46 using System.Linq;\r
  47 using System.Reflection;\r
  48 using System.Threading;\r
  49 using System.Threading.Tasks;\r
  50 using OpenSSL.Crypto;\r
  51 using Pithos.Network;\r
  52 using Alpha = Alphaleonis.Win32.Filesystem;\r
  53 \r
  54 namespace Pithos.Core.Agents\r
  55 {\r
  56     class BlockUpdater\r
  57     {\r
  58         //TODO: Must clean orphaned blocks from the Cache folder.\r
  59         //\r
  60         //The Cache folder may have orphaned blocks. Blocks may be left in the Cache folder because:\r
  61         //1. A download was in progress when the application terminated. These blocks are needed to proceed \r
  62         //  with partial download\r
  63         //2. The application terminated abnormally before the blocks were cleared after a download\r
  64         //3. The server file was deleted before the download completed.\r
  65         //\r
  66         //In #1, we need to keep the blocks. We need to detect the other cases and delete orphans\r
  67         //\r
  68         //Mitigations:\r
  69         // - Delete blocks with no corresponding state\r
  70         // - Check and delete possible orphans when a Deletion is detected\r
  71         // - Add Advanced command "Clear Cache"\r
  72         //\r
  73         //Need a better way to differentiate between cases #2, #3 and #1\r
  74 \r
  75         private static readonly log4net.ILog Log = log4net.LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
  76 \r
  77         public string FilePath { get; private set; }\r
  78         public string RelativePath { get; private set; }\r
  79 \r
  80         public string CachePath { get; private set; }\r
  81 \r
  82         public TreeHash ServerHash { get; private set; }\r
  83 \r
  84         public string TempPath { get; private set; }\r
  85 \r
  86         public bool HasBlocks\r
  87         {\r
  88             get { return _blocks.Count>0; }            \r
  89         }\r
  90 \r
  91         readonly ConcurrentDictionary<long, string> _blocks = new ConcurrentDictionary<long, string>();\r
  92         readonly ConcurrentDictionary<string, string> _orphanBlocks = new ConcurrentDictionary<string, string>();\r
  93 \r
  94         [ContractInvariantMethod]\r
  95         private void Invariants()\r
  96         {\r
  97             Contract.Invariant(Path.IsPathRooted(CachePath));\r
  98             Contract.Invariant(Path.IsPathRooted(FilePath));\r
  99             Contract.Invariant(Path.IsPathRooted(TempPath));\r
 100             Contract.Invariant(!Path.IsPathRooted(RelativePath));\r
 101             Contract.Invariant(_blocks!=null);\r
 102             Contract.Invariant(_orphanBlocks!=null);\r
 103             Contract.Invariant(ServerHash!=null);\r
 104         }\r
 105 \r
 106         public BlockUpdater(string cachePath, string filePath, string relativePath,TreeHash serverHash)\r
 107         {   \r
 108             if (String.IsNullOrWhiteSpace(cachePath))\r
 109                 throw new ArgumentNullException("cachePath");\r
 110             if (!Path.IsPathRooted(cachePath))\r
 111                 throw new ArgumentException("The cachePath must be rooted", "cachePath");\r
 112             \r
 113             if (string.IsNullOrWhiteSpace(filePath))\r
 114                 throw new ArgumentNullException("filePath");\r
 115             if (!Path.IsPathRooted(filePath))\r
 116                 throw new ArgumentException("The filePath must be rooted", "filePath");\r
 117             \r
 118             if (string.IsNullOrWhiteSpace(relativePath))\r
 119                 throw new ArgumentNullException("relativePath");\r
 120             if (Path.IsPathRooted(relativePath))\r
 121                 throw new ArgumentException("The relativePath must NOT be rooted", "relativePath");\r
 122 \r
 123             if (serverHash == null)\r
 124                 throw new ArgumentNullException("serverHash");\r
 125             Contract.EndContractBlock();\r
 126 \r
 127             CachePath=cachePath;\r
 128             FilePath = filePath;\r
 129             RelativePath=relativePath;\r
 130             ServerHash = serverHash;\r
 131             //The file will be stored in a temporary location while downloading with an extension .download\r
 132             TempPath = Path.Combine(CachePath, RelativePath + ".download");\r
 133             \r
 134             //Need to calculate the directory path because RelativePath may include folders\r
 135             var directoryPath = Path.GetDirectoryName(TempPath);            \r
 136             //directoryPath CAN be null if TempPath is a root path\r
 137             if (String.IsNullOrWhiteSpace(directoryPath))\r
 138                 throw new ArgumentException("TempPath");\r
 139             //CachePath was absolute so directoryPath is absolute too\r
 140             Contract.Assume(Path.IsPathRooted(directoryPath));\r
 141             \r
 142             if (!Directory.Exists(directoryPath))\r
 143                 Directory.CreateDirectory(directoryPath);\r
 144 \r
 145             LoadOrphans(directoryPath);\r
 146         }\r
 147 \r
 148         private void LoadOrphans(string directoryPath)\r
 149         {\r
 150             if (string.IsNullOrWhiteSpace(directoryPath))\r
 151                 throw new ArgumentNullException("directoryPath");\r
 152             if (!Path.IsPathRooted(directoryPath))\r
 153                 throw new ArgumentException("The directoryPath must be rooted", "directoryPath");\r
 154             if (ServerHash==null)\r
 155                 throw new InvalidOperationException("ServerHash wasn't initialized");\r
 156             Contract.EndContractBlock();\r
 157 \r
 158             var fileNamename = Path.GetFileName(FilePath);\r
 159             var orphans = Directory.GetFiles(directoryPath, fileNamename + ".*");\r
 160             foreach (var orphan in orphans)\r
 161             {\r
 162                 using (var hasher = new MessageDigestContext(MessageDigest.CreateByName(ServerHash.BlockHash)))                \r
 163                 {\r
 164                     hasher.Init();\r
 165                     var buffer=File.ReadAllBytes(orphan);\r
 166                     //The server truncates nulls before calculating hashes, have to do the same\r
 167                     //Find the last non-null byte, starting from the end\r
 168                     var lastByteIndex = Array.FindLastIndex(buffer, buffer.Length-1, aByte => aByte != 0);\r
 169                     //lastByteIndex may be -1 if the file was empty. We don't want to use that block file\r
 170                     if (lastByteIndex >= 0)\r
 171                     {\r
 172                         byte[] block;\r
 173                         if (lastByteIndex == buffer.Length - 1)\r
 174                             block = buffer;\r
 175                         else\r
 176                         {\r
 177                             block=new byte[lastByteIndex+1];\r
 178                             Buffer.BlockCopy(buffer,0,block,0,lastByteIndex+1);\r
 179                         }\r
 180                         var binHash = hasher.Digest(block);\r
 181                         var hash = binHash.ToHashString();\r
 182                         _orphanBlocks[hash] = orphan;\r
 183                     }\r
 184                 }\r
 185             }\r
 186         }\r
 187 \r
 188 \r
 189         public void Commit()\r
 190         {\r
 191             if (String.IsNullOrWhiteSpace(FilePath))\r
 192                 throw new InvalidOperationException("FilePath is empty");\r
 193             if (String.IsNullOrWhiteSpace(TempPath))\r
 194                 throw new InvalidOperationException("TempPath is empty");\r
 195             Contract.EndContractBlock();\r
 196 \r
 197             var info = Alpha.Volume.GetVolumeInformation(Alpha.Path.GetPathRoot(FilePath));            \r
 198             var isNTFS = info.FileSystemName.Equals("NTFS");                \r
 199 \r
 200 \r
 201             if (isNTFS && MS.WindowsAPICodePack.Internal.CoreHelpers.RunningOnVista)\r
 202             {\r
 203                 ApplyBlocksWithTransaction();\r
 204             }\r
 205             else\r
 206             {\r
 207                 ApplyBlocksWithCopy();\r
 208             }\r
 209             ClearBlocks();\r
 210         }\r
 211 \r
 212         private void ApplyBlocksWithTransaction()\r
 213         {\r
 214             var targetFile = FilePath;\r
 215             using (var tx = new Alpha.KernelTransaction())\r
 216             {\r
 217                 //Retry if the file is still open\r
 218                     using (var stream = Alpha.File.Open(tx, targetFile, \r
 219                         Alpha.FileMode.OpenOrCreate,Alpha.FileAccess.Write))\r
 220                     {\r
 221                         stream.SetLength(ServerHash.Bytes);\r
 222                         foreach (var block in _blocks)\r
 223                         {\r
 224                             var blockPath = block.Value;\r
 225                             var blockIndex = block.Key;\r
 226                             //Retry if we can't open the block immediatelly\r
 227                             for (int i = 0; i < 3; i++)\r
 228                             {\r
 229                                 try\r
 230                                 {\r
 231                                     using (var blockStream = File.OpenRead(blockPath))\r
 232                                     {\r
 233                                         long offset = blockIndex*ServerHash.BlockSize;\r
 234                                         stream.Seek(offset, SeekOrigin.Begin);\r
 235                                         blockStream.CopyTo(stream);\r
 236                                     }\r
 237                                     break;\r
 238                                 }\r
 239                                 catch (IOException)\r
 240                                 {\r
 241                                     if (i>=2)\r
 242                                         throw;\r
 243                                     Thread.Sleep((i+1)*300);\r
 244                                 }                                \r
 245                             }\r
 246                         }\r
 247                     }\r
 248                 tx.Commit();\r
 249             }\r
 250 \r
 251         }\r
 252 \r
 253         private void ApplyBlocksWithCopy()\r
 254         {\r
 255             //Copy the file to a temporary location. Changes will be made to the\r
 256             //temporary file, then it will replace the original file\r
 257             if (File.Exists(FilePath))\r
 258                 File.Copy(FilePath, TempPath, true);\r
 259 \r
 260             //Set the size of the file to the size specified in the treehash\r
 261             //This will also create an empty file if the file doesn't exist                        \r
 262 \r
 263             ApplyBlocks(TempPath);\r
 264             SwapFiles();\r
 265         }\r
 266 \r
 267         private void ApplyBlocks(string targetFile)\r
 268         {\r
 269             SetFileSize(targetFile, ServerHash.Bytes);\r
 270 \r
 271             //Update the temporary file with the data from the blocks\r
 272             using (var stream = File.OpenWrite(targetFile))\r
 273             {\r
 274                 foreach (var block in _blocks)\r
 275                 {\r
 276                     var blockPath = block.Value;\r
 277                     var blockIndex = block.Key;\r
 278                     using (var blockStream = File.OpenRead(blockPath))\r
 279                     {\r
 280                         long offset = blockIndex*ServerHash.BlockSize;\r
 281                         stream.Seek(offset, SeekOrigin.Begin);\r
 282                         blockStream.CopyTo(stream);\r
 283                     }\r
 284                 }\r
 285             }\r
 286         }\r
 287 \r
 288         private void SwapFiles()\r
 289         {\r
 290             if (String.IsNullOrWhiteSpace(FilePath))\r
 291                 throw new InvalidOperationException("FilePath is empty");\r
 292             if (String.IsNullOrWhiteSpace(TempPath))\r
 293                 throw new InvalidOperationException("TempPath is empty");            \r
 294             Contract.EndContractBlock();\r
 295 \r
 296             if (File.Exists(FilePath))\r
 297                 File.Replace(TempPath, FilePath, null, true);\r
 298             else\r
 299             {\r
 300                 var targetDirectory = Path.GetDirectoryName(FilePath);\r
 301                 if (!Directory.Exists(targetDirectory))\r
 302                     Directory.CreateDirectory(targetDirectory);\r
 303                 File.Move(TempPath, FilePath);\r
 304             }\r
 305         }\r
 306 \r
 307         private void ClearBlocks()\r
 308         {\r
 309             if (Log.IsDebugEnabled)\r
 310                 Log.DebugFormat("Clearing blocks for {0}",this.FilePath);\r
 311             //Get all the the block paths, orphan or not\r
 312             var paths= _blocks.Select(pair => pair.Value)\r
 313                           .Union(_orphanBlocks.Select(pair => pair.Value));\r
 314             foreach (var filePath in paths)\r
 315             {\r
 316                 File.Delete(filePath);\r
 317             }\r
 318 \r
 319             File.Delete(TempPath);\r
 320             _blocks.Clear();\r
 321             _orphanBlocks.Clear();\r
 322         }\r
 323 \r
 324         //Change the file's size, possibly truncating or adding to it\r
 325         private  void SetFileSize(string filePath, long fileSize)\r
 326         {\r
 327             if (String.IsNullOrWhiteSpace(filePath))\r
 328                 throw new ArgumentNullException("filePath");\r
 329             if (!Path.IsPathRooted(filePath))\r
 330                 throw new ArgumentException("The filePath must be rooted", "filePath");\r
 331             if (fileSize < 0)\r
 332                 throw new ArgumentOutOfRangeException("fileSize");\r
 333             Contract.EndContractBlock();\r
 334 \r
 335             using (var stream = File.Open(filePath, FileMode.OpenOrCreate, FileAccess.Write))\r
 336             {\r
 337                 stream.SetLength(fileSize);\r
 338             }\r
 339         }\r
 340 \r
 341        /* //Check whether we should copy the local file to a temp path        \r
 342         private  bool ShouldCopy(string localPath, string tempPath)\r
 343         {\r
 344             //No need to copy if there is no file\r
 345             if (!File.Exists(localPath))\r
 346                 return false;\r
 347 \r
 348             //If there is no temp file, go ahead and copy\r
 349             if (!File.Exists(tempPath))\r
 350                 return true;\r
 351 \r
 352             //If there is a temp file and is newer than the actual file, don't copy\r
 353             var localLastWrite = File.GetLastWriteTime(localPath);\r
 354             var tempLastWrite = File.GetLastWriteTime(tempPath);\r
 355 \r
 356             //This could mean there is an interrupted download in progress\r
 357             return (tempLastWrite < localLastWrite);\r
 358         }*/\r
 359 \r
 360 \r
 361         public bool UseOrphan(long blockIndex, string blockHash)\r
 362         {\r
 363             string blockPath=null;\r
 364             if (_orphanBlocks.TryGetValue(blockHash,out blockPath))\r
 365             {\r
 366                 _blocks[blockIndex] = blockPath;\r
 367                 return true;\r
 368             }\r
 369             return false;\r
 370         }\r
 371 \r
 372         public Task StoreBlock(long blockIndex,byte[] buffer)\r
 373         {\r
 374             var blockPath = String.Format("{0}.{1:000000}", TempPath, blockIndex);\r
 375             _blocks[blockIndex] = blockPath;\r
 376             //Remove any orphan files\r
 377             if (File.Exists(blockPath))\r
 378                 File.Delete(blockPath);\r
 379 \r
 380             return FileAsync.WriteAllBytes(blockPath, buffer);\r
 381         }\r
 382 \r
 383        \r
 384 \r
 385     }\r
 386 }\r