Synnefo » Pithos MS Client

using System;

using System.Collections.Concurrent;

using System.Collections.Generic;

using System.Diagnostics;

using System.IO;

using System.Linq;

using System.Reflection;

using System.Threading;

using System.Threading.Tasks;

using Pithos.Interfaces;

using Pithos.Network;

using log4net;

namespace Pithos.Core.Agents

{

    /// <summary>

    /// Creates a set of state tuples from the local files, cloud files and state database entries

    /// </summary>

    class TupleBuilder

    {

        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);

        private readonly string _emptyGuid = Guid.Empty.ToString();

        private readonly CancellationToken _token;

        private readonly IStatusKeeper _statusKeeper;

        private readonly IStatusNotification _statusNotification;

        private readonly IPithosSettings _settings;

        public TupleBuilder(CancellationToken token, IStatusKeeper statusKeeper, IStatusNotification statusNotification, IPithosSettings settings)

        {

            _token = token;

            _statusKeeper = statusKeeper;

            _statusNotification = statusNotification;

            _settings = settings;

        }

        /// <summary>

        /// Creates a list of Sync tuples by merging local and cloud files with the locally stored state

        /// </summary>

        /// <param name="infos"></param>

        /// <param name="files"></param>

        /// <param name="states"></param>

        /// <param name="moves"></param>

        /// <returns></returns>

        public IEnumerable<StateTuple> MergeSources(IEnumerable<Tuple<string, ObjectInfo>> infos, IEnumerable<FileSystemInfo> files, List<FileState> states, ConcurrentDictionary<string, MovedEventArgs> moves)

        {

            var tuplesByPath = new Dictionary<string, StateTuple>();

            //Fill the tuples with the local files

            CreateTuplesFromFiles(files, states, moves, tuplesByPath);

            //Merge the file tuples with the local states, creating new tuples for states that have no matching files

            MergeLocalStates(states, moves, tuplesByPath);

            MergeCloudFiles(infos, tuplesByPath);

            DetectLocalMoves(tuplesByPath);

            var tuples = tuplesByPath.Values;

            //Sync algorithm fallout: There are multiple ways we can reach a situation where a state without a checksum exists

            //1: The application stopped/crashed while downloading a file. The file's entry was created when the download started. When the application restarts,

            //  it finds no local file, a server file and a null state -> C: NULL L: NULL but exists, S: Some

            //  It can be fixed by NOT creating a local state if the file doesn't already exist, or adding extra info to mark this as a result of an upload

            //2: A new file is added but the app stops/crashes after uploading finishes but before the entry gets updated and the user deletes the file. The file's entry was created. When the app restarts,

            //  it finds no local file, a server file and a null state -> C: NULL L: NULL but exists, S: Some

            //  

            var brokenTuples =( from tuple in tuples

                               where tuple.FileState != null && tuple.FileState.Checksum == null

                                    && tuple.ObjectInfo != null && (tuple.FileInfo == null || !tuple.FileInfo.Exists)

                               select tuple).ToList();

            var actualTuples = tuples.Except(brokenTuples).ToList();

            Debug.Assert(actualTuples.All(t => t.HashesValid()));

            foreach (var tuple in brokenTuples)

            {

                _statusKeeper.SetFileState(tuple.FileState.FilePath,

                    FileStatus.Conflict, FileOverlayStatus.Conflict, "FileState without checksum encountered for server object missing from disk");

            }

            return actualTuples;

        }

        /// <summary>

        /// 

        /// </summary>

        /// <param name="tuplesByPath"></param>

        /// <remarks>

        /// Local moves will appear as two tuples with the same hashes in different locations:

        /// - The FROM tuple will have a NULL C, and a filled L and S value

        /// - The TO tuple will only have a C hash value equal to FROM's and empty L,S

        /// The FROM tuple should be removed and the TO tuple should be updated to reflect that it's a MOVE operation.

        /// This should happen only if there are EXACTLY two tuples with the same hash

        /// </remarks>

        private void DetectLocalMoves(Dictionary<string, StateTuple> tuplesByPath)

        {

            //Newly created fiels are candidate TOs

            var creates = tuplesByPath.Values.Where(t=> t.FileInfo is FileInfo && t.C!=null && t.L == null && t.S == null);

            //Newly deleted files are candidate FROMs

            var deletes = tuplesByPath.Values.Where(t =>t.NullSafe(t1=>t1.FileState).NullSafe(s=>!s.IsFolder) &&  t.C == null && t.L != null && t.L==t.S);

            var moves = (from tuple in creates

                        let froms = deletes.Where(d => d.L == tuple.C)

                        where froms.Count() == 1

                        select new {To = tuple, From = froms.Single()}).ToList();

            if (!moves.Any())

                return;

            foreach (var move in moves)

            {

                tuplesByPath.Remove(move.From.FilePath);

                move.To.OldFullPath = move.From.FilePath;

                move.To.NewFullPath = move.To.FilePath;

                move.To.OldChecksum = move.From.L;

                move.To.FileState = move.From.FileState;

                move.To.ObjectInfo = move.From.ObjectInfo;

            }

        }

        private static void CreateTuplesFromFiles(IEnumerable<FileSystemInfo> files, List<FileState> states, ConcurrentDictionary<string, MovedEventArgs> moves, Dictionary<string, StateTuple> tuplesByPath)

        {

            foreach (var info in files)

            {

                var tuple = new StateTuple(info);

                //Is this the target of a move event?

                var moveArg =

                    moves.Values.FirstOrDefault(

                        arg => info.FullName.Equals(arg.FullPath, StringComparison.InvariantCultureIgnoreCase)

                               || info.FullName.IsAtOrBelow(arg.FullPath));

                if (moveArg != null)

                {

                    tuple.NewFullPath = info.FullName;

                    var relativePath = info.AsRelativeTo(moveArg.FullPath);

                    tuple.OldFullPath = Path.Combine(moveArg.OldFullPath, relativePath);

                    tuple.OldChecksum = states.FirstOrDefault(

                        st => st.FilePath.Equals(tuple.OldFullPath, StringComparison.InvariantCultureIgnoreCase))

                        .NullSafe(st => st.Checksum);

                }

                tuplesByPath[tuple.FilePath] = tuple;

            }

        }

        private void MergeLocalStates(IEnumerable<FileState> states, ConcurrentDictionary<string, MovedEventArgs> moves, Dictionary<string, StateTuple> tuplesByPath)

        {

            //For files that have state

            foreach (var state in states)

            {

                StateTuple hashTuple;

                if (tuplesByPath.TryGetValue(state.FilePath, out hashTuple))

                {

                    hashTuple.FileState = state;

                    UpdateHashes(hashTuple);

                }

                else if (moves.ContainsKey(state.FilePath) &&

                         tuplesByPath.TryGetValue(moves[state.FilePath].FullPath, out hashTuple))

                {

                    hashTuple.FileState = state;

                    UpdateHashes(hashTuple);

                }

                else

                {

                    var fsInfo = FileInfoExtensions.FromPath(state.FilePath);

                    hashTuple = new StateTuple { FileInfo = fsInfo, FileState = state };

                    //Is the source of a moved item?

                    var moveArg =

                        moves.Values.FirstOrDefault(

                            arg => state.FilePath.Equals(arg.OldFullPath, StringComparison.InvariantCultureIgnoreCase)

                                   || state.FilePath.IsAtOrBelow(arg.OldFullPath));

                    if (moveArg != null)

                    {

                        var relativePath = state.FilePath.AsRelativeTo(moveArg.OldFullPath);

                        hashTuple.NewFullPath = Path.Combine(moveArg.FullPath, relativePath);

                        hashTuple.OldFullPath = state.FilePath;

                        //Do we have the old MD5?

                        //hashTuple.OldMD5 = state.LastMD5;

                    }

                    tuplesByPath[state.FilePath] = hashTuple;

                }

            }

            //for files that don't have state

            var statelessTuples = tuplesByPath.Values.Where(t => t.FileState == null).ToArray();

            //If there are too many stateless tuples, update them in parallel

            if (statelessTuples.Length > 20)

                Parallel.ForEach(statelessTuples, UpdateHashes);

            else

                statelessTuples.ApplyAction(UpdateHashes);

        }

        /// <summary>

        /// Update the tuple with the file's hashes, avoiding calculation if the file is unchanged

        /// </summary>

        /// <param name="hashTuple"></param>

        /// <remarks>

        /// The function first checks the file's size and last write date to see if there are any changes. If there are none,

        /// the file's stored hashes are used.

        /// Otherwise, MD5 is calculated first to ensure there are no changes. If MD5 is different, the Merkle hash is calculated

        /// </remarks>

        private void UpdateHashes(StateTuple hashTuple)

        {

            try

            {

                var state = hashTuple.NullSafe(s => s.FileState);

                var storedHash = state.NullSafe(s => s.Checksum);

                var storedHashes = state.NullSafe(s => s.Hashes);

                //var storedMD5 = state.NullSafe(s => s.LastMD5);

                var storedDate = state.NullSafe(s => s.LastWriteDate) ?? DateTime.MinValue;

                var storedLength = state.NullSafe(s => s.LastLength);

                //var md5Hash = Signature.MD5_EMPTY;                

                var merkle = TreeHash.Empty;

                if (hashTuple.FileInfo is FileInfo)

                {

                    var file = (FileInfo)hashTuple.FileInfo.WithProperCapitalization();

                    //Attributes unchanged?

                    //LastWriteTime is only accurate to the second

                    var unchangedAttributes = file.LastWriteTime - storedDate < TimeSpan.FromSeconds(1)

                        && storedLength == file.Length;

                    //Attributes appear unchanged but the file length doesn't match the stored hash ?

                    var nonEmptyMismatch = unchangedAttributes &&

                        (file.Length == 0 ^ storedHash == Signature.MERKLE_EMPTY);

                    //Missing hashes for NON-EMPTY hash ?

                    var missingHashes = storedHash != Signature.MERKLE_EMPTY &&

                        String.IsNullOrWhiteSpace(storedHashes);

                    //Unchanged attributes but changed MD5 

                    //Short-circuiting ensures MD5 is computed only if the attributes are changed

                    //var md5Mismatch = (!unchangedAttributes && file.ComputeShortHash(StatusNotification) != storedMD5);

                    //If the attributes are unchanged but the Merkle doesn't match the size,

                    //or the attributes and the MD5 hash have changed, 

                    //or the hashes are missing but the tophash is NOT empty, we need to recalculate

                    //

                    //Otherwise we load the hashes from state

                    if (!unchangedAttributes || nonEmptyMismatch || missingHashes)

                        merkle = RecalculateTreehash(file);

                    else

                    {

                        merkle = TreeHash.Parse(hashTuple.FileState.Hashes);

                        //merkle.MD5 = storedMD5;

                    }

                    //md5Hash = merkle.MD5;

                }

                //hashTuple.MD5 = md5Hash;

                //Setting Merkle also updates C

                hashTuple.Merkle = merkle;

            }

            catch (IOException)

            {

                hashTuple.Locked = true;

            }

        }

        /// <summary>

        /// Recalculate a file's treehash and md5 and update the database

        /// </summary>

        /// <param name="file"></param>

        /// <returns></returns>

        private TreeHash RecalculateTreehash(FileInfo file)

        {

            var progress = new Progress<HashProgress>(d => _statusNotification.Notify(

                                                    new StatusNotification(String.Format("Hashing {0:p} of {1}", d.Percentage, file.Name))));

            var merkle = Signature.CalculateTreeHash(file, _statusKeeper.BlockSize, _statusKeeper.BlockHash,

                _settings.HashingParallelism, _token, progress);

            _statusKeeper.UpdateFileHashes(file, merkle);

            return merkle;

        }

        private void MergeCloudFiles(IEnumerable<Tuple<string, ObjectInfo>> infos, Dictionary<string, StateTuple> tuplesByPath)

        {

            var tuplesById = tuplesByPath.Values

                .Where(tuple => tuple.FileState != null && tuple.FileState.ObjectID != null)

                .ToDictionary(tuple => tuple.FileState.ObjectID, tuple => tuple); //new Dictionary<Guid, StateTuple>();

            foreach (var info in infos)

            {

                StateTuple hashTuple;

                var filePath = info.Item1;

                var objectInfo = info.Item2;

                var objectId = objectInfo.UUID;

                if (objectId != _emptyGuid && tuplesById.TryGetValue(objectId, out hashTuple))

                {

                    //This will handle renamed objects

                    hashTuple.ObjectInfo = objectInfo;

                }

                else if (tuplesByPath.TryGetValue(filePath, out hashTuple))

                {

                    hashTuple.ObjectInfo = objectInfo;

                }

                else

                {

                    var fsInfo = FileInfoExtensions.FromPath(filePath);

                    hashTuple = new StateTuple { FileInfo = fsInfo, ObjectInfo = objectInfo };

                    tuplesByPath[filePath] = hashTuple;

                    if (objectInfo.UUID != _emptyGuid)

                        tuplesById[objectInfo.UUID] = hashTuple;

                }

            }

        }

    }

}