code.grnet.gr Git - pithos-ms-client/blob - trunk/Pithos.Core/Agents/BlockUpdater.cs

   1 #region
   2 /* -----------------------------------------------------------------------
   3  * <copyright file="BlockUpdater.cs" company="GRNet">
   4  *
   5  * Copyright 2011-2012 GRNET S.A. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or
   8  * without modification, are permitted provided that the following
   9  * conditions are met:
  10  *
  11  *   1. Redistributions of source code must retain the above
  12  *      copyright notice, this list of conditions and the following
  13  *      disclaimer.
  14  *
  15  *   2. Redistributions in binary form must reproduce the above
  16  *      copyright notice, this list of conditions and the following
  17  *      disclaimer in the documentation and/or other materials
  18  *      provided with the distribution.
  19  *
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
  22  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  28  * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  31  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  * The views and conclusions contained in the software and
  35  * documentation are those of the authors and should not be
  36  * interpreted as representing official policies, either expressed
  37  * or implied, of GRNET S.A.
  38  * </copyright>
  39  * -----------------------------------------------------------------------
  40  */
  41 #endregion
  42 using System;
  43 using System.Collections.Concurrent;
  44 using System.Collections.Generic;
  45 using System.Diagnostics.Contracts;
  46 using System.IO;
  47 using System.Linq;
  48 using System.Reflection;
  49 using System.Security.Cryptography;
  50 using System.Text;
  51 using System.Threading.Tasks;
  52 using Pithos.Network;
  53
  54 namespace Pithos.Core.Agents
  55 {
  56     class BlockUpdater
  57     {
  58         //TODO: Must clean orphaned blocks from the Cache folder.
  59         //
  60         //The Cache folder may have orphaned blocks. Blocks may be left in the Cache folder because:
  61         //1. A download was in progress when the application terminated. These blocks are needed to proceed
  62         //  with partial download
  63         //2. The application terminated abnormally before the blocks were cleared after a download
  64         //3. The server file was deleted before the download completed.
  65         //
  66         //In #1, we need to keep the blocks. We need to detect the other cases and delete orphans
  67         //
  68         //Mitigations:
  69         // - Delete blocks with no corresponding state
  70         // - Check and delete possible orphans when a Deletion is detected
  71         // - Add Advanced command "Clear Cache"
  72         //
  73         //Need a better way to differentiate between cases #2, #3 and #1
  74
  75         private static readonly log4net.ILog Log = log4net.LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  76
  77         public string FilePath { get; private set; }
  78         public string RelativePath { get; private set; }
  79
  80         public string CachePath { get; private set; }
  81
  82         public TreeHash ServerHash { get; private set; }
  83
  84         public string TempPath { get; private set; }
  85
  86         public bool HasBlocks
  87         {
  88             get { return _blocks.Count>0; }
  89         }
  90
  91         readonly ConcurrentDictionary<int, string> _blocks = new ConcurrentDictionary<int, string>();
  92         readonly ConcurrentDictionary<string, string> _orphanBlocks = new ConcurrentDictionary<string, string>();
  93
  94         [ContractInvariantMethod]
  95         private void Invariants()
  96         {
  97             Contract.Invariant(Path.IsPathRooted(CachePath));
  98             Contract.Invariant(Path.IsPathRooted(FilePath));
  99             Contract.Invariant(Path.IsPathRooted(TempPath));
 100             Contract.Invariant(!Path.IsPathRooted(RelativePath));
 101             Contract.Invariant(_blocks!=null);
 102             Contract.Invariant(_orphanBlocks!=null);
 103             Contract.Invariant(ServerHash!=null);
 104         }
 105
 106         public BlockUpdater(string cachePath, string filePath, string relativePath,TreeHash serverHash)
 107         {
 108             if (String.IsNullOrWhiteSpace(cachePath))
 109                 throw new ArgumentNullException("cachePath");
 110             if (!Path.IsPathRooted(cachePath))
 111                 throw new ArgumentException("The cachePath must be rooted", "cachePath");
 112
 113             if (string.IsNullOrWhiteSpace(filePath))
 114                 throw new ArgumentNullException("filePath");
 115             if (!Path.IsPathRooted(filePath))
 116                 throw new ArgumentException("The filePath must be rooted", "filePath");
 117
 118             if (string.IsNullOrWhiteSpace(relativePath))
 119                 throw new ArgumentNullException("relativePath");
 120             if (Path.IsPathRooted(relativePath))
 121                 throw new ArgumentException("The relativePath must NOT be rooted", "relativePath");
 122
 123             if (serverHash == null)
 124                 throw new ArgumentNullException("serverHash");
 125             Contract.EndContractBlock();
 126
 127             CachePath=cachePath;
 128             FilePath = filePath;
 129             RelativePath=relativePath;
 130             ServerHash = serverHash;
 131             //The file will be stored in a temporary location while downloading with an extension .download
 132             TempPath = Path.Combine(CachePath, RelativePath + ".download");
 133
 134             //Need to calculate the directory path because RelativePath may include folders
 135             var directoryPath = Path.GetDirectoryName(TempPath);
 136             //directoryPath CAN be null if TempPath is a root path
 137             if (String.IsNullOrWhiteSpace(directoryPath))
 138                 throw new ArgumentException("TempPath");
 139             //CachePath was absolute so directoryPath is absolute too
 140             Contract.Assume(Path.IsPathRooted(directoryPath));
 141
 142             if (!Directory.Exists(directoryPath))
 143                 Directory.CreateDirectory(directoryPath);
 144
 145             LoadOrphans(directoryPath);
 146         }
 147
 148         private void LoadOrphans(string directoryPath)
 149         {
 150             if (string.IsNullOrWhiteSpace(directoryPath))
 151                 throw new ArgumentNullException("directoryPath");
 152             if (!Path.IsPathRooted(directoryPath))
 153                 throw new ArgumentException("The directoryPath must be rooted", "directoryPath");
 154             if (ServerHash==null)
 155                 throw new InvalidOperationException("ServerHash wasn't initialized");
 156             Contract.EndContractBlock();
 157
 158             var fileNamename = Path.GetFileName(FilePath);
 159             var orphans = Directory.GetFiles(directoryPath, fileNamename + ".*");
 160             foreach (var orphan in orphans)
 161             {
 162                 using (HashAlgorithm hasher = HashAlgorithm.Create(ServerHash.BlockHash))
 163                 {
 164                     var buffer=File.ReadAllBytes(orphan);
 165                     //The server truncates nulls before calculating hashes, have to do the same
 166                     //Find the last non-null byte, starting from the end
 167                     var lastByteIndex = Array.FindLastIndex(buffer, buffer.Length-1, aByte => aByte != 0);
 168                     //lastByteIndex may be -1 if the file was empty. We don't want to use that block file
 169                     if (lastByteIndex >= 0)
 170                     {
 171                         var binHash = hasher.ComputeHash(buffer, 0, lastByteIndex);
 172                         var hash = binHash.ToHashString();
 173                         _orphanBlocks[hash] = orphan;
 174                     }
 175                 }
 176             }
 177         }
 178
 179
 180         public void Commit()
 181         {
 182             if (String.IsNullOrWhiteSpace(FilePath))
 183                 throw new InvalidOperationException("FilePath is empty");
 184             if (String.IsNullOrWhiteSpace(TempPath))
 185                 throw new InvalidOperationException("TempPath is empty");
 186             Contract.EndContractBlock();
 187
 188             //Copy the file to a temporary location. Changes will be made to the
 189             //temporary file, then it will replace the original file
 190             if (File.Exists(FilePath))
 191                 File.Copy(FilePath, TempPath, true);
 192
 193             //Set the size of the file to the size specified in the treehash
 194             //This will also create an empty file if the file doesn't exist
 195
 196
 197             SetFileSize(TempPath, ServerHash.Bytes);
 198
 199             //Update the temporary file with the data from the blocks
 200             using (var stream = File.OpenWrite(TempPath))
 201             {
 202                 foreach (var block in _blocks)
 203                 {
 204                     var blockPath = block.Value;
 205                     var blockIndex = block.Key;
 206                     using (var blockStream = File.OpenRead(blockPath))
 207                     {
 208                         long offset = blockIndex*ServerHash.BlockSize;
 209                         stream.Seek(offset, SeekOrigin.Begin);
 210                         blockStream.CopyTo(stream);
 211                     }
 212                 }
 213             }
 214             SwapFiles();
 215
 216             ClearBlocks();
 217         }
 218
 219         private void SwapFiles()
 220         {
 221             if (String.IsNullOrWhiteSpace(FilePath))
 222                 throw new InvalidOperationException("FilePath is empty");
 223             if (String.IsNullOrWhiteSpace(TempPath))
 224                 throw new InvalidOperationException("TempPath is empty");
 225             Contract.EndContractBlock();
 226
 227             if (File.Exists(FilePath))
 228                 File.Replace(TempPath, FilePath, null, true);
 229             else
 230             {
 231                 var targetDirectory = Path.GetDirectoryName(FilePath);
 232                 if (!Directory.Exists(targetDirectory))
 233                     Directory.CreateDirectory(targetDirectory);
 234                 File.Move(TempPath, FilePath);
 235             }
 236         }
 237
 238         private void ClearBlocks()
 239         {
 240             if (Log.IsDebugEnabled)
 241                 Log.DebugFormat("Clearing blocks for {0}",this.FilePath);
 242             //Get all the the block paths, orphan or not
 243             var paths= _blocks.Select(pair => pair.Value)
 244                           .Union(_orphanBlocks.Select(pair => pair.Value));
 245             foreach (var filePath in paths)
 246             {
 247                 File.Delete(filePath);
 248             }
 249
 250             File.Delete(TempPath);
 251             _blocks.Clear();
 252             _orphanBlocks.Clear();
 253         }
 254
 255         //Change the file's size, possibly truncating or adding to it
 256         private  void SetFileSize(string filePath, long fileSize)
 257         {
 258             if (String.IsNullOrWhiteSpace(filePath))
 259                 throw new ArgumentNullException("filePath");
 260             if (!Path.IsPathRooted(filePath))
 261                 throw new ArgumentException("The filePath must be rooted", "filePath");
 262             if (fileSize < 0)
 263                 throw new ArgumentOutOfRangeException("fileSize");
 264             Contract.EndContractBlock();
 265
 266             using (var stream = File.Open(filePath, FileMode.OpenOrCreate, FileAccess.Write))
 267             {
 268                 stream.SetLength(fileSize);
 269             }
 270         }
 271
 272        /* //Check whether we should copy the local file to a temp path
 273         private  bool ShouldCopy(string localPath, string tempPath)
 274         {
 275             //No need to copy if there is no file
 276             if (!File.Exists(localPath))
 277                 return false;
 278
 279             //If there is no temp file, go ahead and copy
 280             if (!File.Exists(tempPath))
 281                 return true;
 282
 283             //If there is a temp file and is newer than the actual file, don't copy
 284             var localLastWrite = File.GetLastWriteTime(localPath);
 285             var tempLastWrite = File.GetLastWriteTime(tempPath);
 286
 287             //This could mean there is an interrupted download in progress
 288             return (tempLastWrite < localLastWrite);
 289         }*/
 290
 291
 292         public bool UseOrphan(int blockIndex, string blockHash)
 293         {
 294             string blockPath=null;
 295             if (_orphanBlocks.TryGetValue(blockHash,out blockPath))
 296             {
 297                 _blocks[blockIndex] = blockPath;
 298                 return true;
 299             }
 300             return false;
 301         }
 302
 303         public Task StoreBlock(int blockIndex,byte[] buffer)
 304         {
 305             var blockPath = String.Format("{0}.{1:000000}", TempPath, blockIndex);
 306             _blocks[blockIndex] = blockPath;
 307             //Remove any orphan files
 308             if (File.Exists(blockPath))
 309                 File.Delete(blockPath);
 310
 311             return FileAsync.WriteAllBytes(blockPath, buffer);
 312         }
 313
 314
 315
 316     }
 317 }