Multiple changes:
authorPanagiotis Kanavos <pkanavos@gmail.com>
Tue, 6 Mar 2012 11:19:09 +0000 (13:19 +0200)
committerPanagiotis Kanavos <pkanavos@gmail.com>
Tue, 6 Mar 2012 11:19:09 +0000 (13:19 +0200)
* Calculate hashes in place
* Use SHA1 to detect local changes before starting the Merkle hash calculation
* Fixes to renaming, downloading behavior
* Changes to logging

29 files changed:
trunk/Pithos.Client.WPF/Configuration/PithosSettings.cs
trunk/Pithos.Client.WPF/Pithos.Client.WPF.csproj
trunk/Pithos.Client.WPF/Shell/ShellViewModel.cs
trunk/Pithos.Client.WPF/app.config
trunk/Pithos.Core.Test/MockStatusKeeper.cs
trunk/Pithos.Core/Agents/Agent.cs
trunk/Pithos.Core/Agents/AsyncCollection.cs [new file with mode: 0644]
trunk/Pithos.Core/Agents/AsyncManualResetEvent.cs
trunk/Pithos.Core/Agents/AsyncSemaphore.cs [new file with mode: 0644]
trunk/Pithos.Core/Agents/BlockExtensions.cs
trunk/Pithos.Core/Agents/BlockUpdater.cs
trunk/Pithos.Core/Agents/CloudTransferAction.cs
trunk/Pithos.Core/Agents/FileAgent.cs
trunk/Pithos.Core/Agents/FileSystemWatcherAdapter.cs
trunk/Pithos.Core/Agents/NetworkAgent.cs
trunk/Pithos.Core/Agents/PollAgent.cs
trunk/Pithos.Core/Agents/StatusAgent.cs
trunk/Pithos.Core/Agents/WorkflowAgent.cs
trunk/Pithos.Core/FileState.cs
trunk/Pithos.Core/IStatusKeeper.cs
trunk/Pithos.Core/IStatusNotification.cs
trunk/Pithos.Core/Pithos.Core.csproj
trunk/Pithos.Core/PithosMonitor.cs
trunk/Pithos.Core/WorkflowState.cs
trunk/Pithos.Interfaces/IStatusChecker.cs
trunk/Pithos.Interfaces/Pithos.Interfaces.csproj
trunk/Pithos.Network/Pithos.Network.csproj
trunk/Pithos.Network/Signature.cs
trunk/Pithos.sln

index 8147f6f..d35db40 100644 (file)
@@ -231,5 +231,11 @@ namespace Pithos.Client.WPF.Configuration
         {
             _settings.Reload();
         }
+
+        public void Reset()
+        {
+            _settings.Reset();
+            _settings.Save();
+        }
     }
 }
index f7af1b8..641c76c 100644 (file)
@@ -30,6 +30,7 @@
     <IsWebBootstrapper>false</IsWebBootstrapper>
     <UseApplicationTrust>false</UseApplicationTrust>
     <BootstrapperEnabled>true</BootstrapperEnabled>
+    <CodeContractsAssemblyMode>0</CodeContractsAssemblyMode>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
     <PlatformTarget>x86</PlatformTarget>
     <CodeAnalysisIgnoreBuiltInRuleSets>false</CodeAnalysisIgnoreBuiltInRuleSets>
     <CodeAnalysisRuleDirectories>;C:\Program Files (x86)\Microsoft Visual Studio 10.0\Team Tools\Static Analysis Tools\FxCop\\Rules</CodeAnalysisRuleDirectories>
     <CodeAnalysisIgnoreBuiltInRules>false</CodeAnalysisIgnoreBuiltInRules>
+    <CodeContractsEnableRuntimeChecking>True</CodeContractsEnableRuntimeChecking>
+    <CodeContractsRuntimeOnlyPublicSurface>False</CodeContractsRuntimeOnlyPublicSurface>
+    <CodeContractsRuntimeThrowOnFailure>True</CodeContractsRuntimeThrowOnFailure>
+    <CodeContractsRuntimeCallSiteRequires>False</CodeContractsRuntimeCallSiteRequires>
+    <CodeContractsRuntimeSkipQuantifiers>False</CodeContractsRuntimeSkipQuantifiers>
+    <CodeContractsRunCodeAnalysis>True</CodeContractsRunCodeAnalysis>
+    <CodeContractsNonNullObligations>True</CodeContractsNonNullObligations>
+    <CodeContractsBoundsObligations>True</CodeContractsBoundsObligations>
+    <CodeContractsArithmeticObligations>True</CodeContractsArithmeticObligations>
+    <CodeContractsEnumObligations>True</CodeContractsEnumObligations>
+    <CodeContractsRedundantAssumptions>True</CodeContractsRedundantAssumptions>
+    <CodeContractsRunInBackground>False</CodeContractsRunInBackground>
+    <CodeContractsShowSquigglies>True</CodeContractsShowSquigglies>
+    <CodeContractsUseBaseLine>False</CodeContractsUseBaseLine>
+    <CodeContractsEmitXMLDocs>False</CodeContractsEmitXMLDocs>
+    <CodeContractsCustomRewriterAssembly />
+    <CodeContractsCustomRewriterClass />
+    <CodeContractsLibPaths />
+    <CodeContractsExtraRewriteOptions />
+    <CodeContractsExtraAnalysisOptions />
+    <CodeContractsBaseLineFile />
+    <CodeContractsCacheAnalysisResults>True</CodeContractsCacheAnalysisResults>
+    <CodeContractsRuntimeCheckingLevel>Full</CodeContractsRuntimeCheckingLevel>
+    <CodeContractsReferenceAssembly>%28none%29</CodeContractsReferenceAssembly>
+    <CodeContractsAnalysisWarningLevel>0</CodeContractsAnalysisWarningLevel>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Premium Debug|AnyCPU'">
     <DebugSymbols>true</DebugSymbols>
index 05a794d..ded5171 100644 (file)
@@ -535,6 +535,7 @@ namespace Pithos.Client.WPF {
                                monitor.Pause = !monitor.Pause;
                                isPaused = monitor.Pause;
                        }
+                        
 
                        PauseSyncCaption = isPaused ? "Resume syncing" : "Pause syncing";
                        var iconKey = isPaused? "TraySyncPaused" : "TrayInSynch";
@@ -557,31 +558,53 @@ namespace Pithos.Client.WPF {
                private readonly Dictionary<PithosStatus, StatusInfo> _iconNames = new List<StatusInfo>
                        {
                                new StatusInfo(PithosStatus.InSynch, "All files up to date", "TrayInSynch"),
-                               new StatusInfo(PithosStatus.Syncing, "Syncing Files", "TraySynching"),
+                               new StatusInfo(PithosStatus.PollSyncing, "Polling Files", "TraySynching"),
+                new StatusInfo(PithosStatus.LocalSyncing, "Syncing Files", "TraySynching"),
                                new StatusInfo(PithosStatus.SyncPaused, "Sync Paused", "TraySyncPaused")
                        }.ToDictionary(s => s.Status);
 
                readonly IWindowManager _windowManager;
                
+        //private int _syncCount=0;
+
+
+        private PithosStatus _pithosStatus = PithosStatus.Disconnected;
+
+        public void SetPithosStatus(PithosStatus status)
+        {
+            if (_pithosStatus == PithosStatus.LocalSyncing && status == PithosStatus.PollComplete)
+                return;
+            if (_pithosStatus == PithosStatus.PollSyncing && status == PithosStatus.LocalComplete)
+                return;
+            if (status == PithosStatus.LocalComplete || status == PithosStatus.PollComplete)
+                _pithosStatus = PithosStatus.InSynch;
+            else
+                _pithosStatus = status;
+            UpdateStatus();
+        }
+
+        public void SetPithosStatus(PithosStatus status,string message)
+        {
+            StatusMessage = message;
+            SetPithosStatus(status);
+        }
+
+
 
                ///<summary>
                /// Updates the visual status indicators of the application depending on status changes, e.g. icon, stat                
                ///</summary>
                public void UpdateStatus()
                {
-                       var pithosStatus = _statusChecker.GetPithosStatus();
 
-                       if (_iconNames.ContainsKey(pithosStatus))
+                       if (_iconNames.ContainsKey(_pithosStatus))
                        {
-                               var info = _iconNames[pithosStatus];
+                               var info = _iconNames[_pithosStatus];
                                StatusIcon = String.Format(@"../Images/{0}.ico", info.IconName);
-
-
-
-                               StatusMessage = String.Format("Pithos {0}\r\n{1}", _fileVersion.Value.FileVersion,info.StatusText);
                        }
-                       
-                       //_events.Publish(new Notification { Title = "Start", Message = "Start Monitoring", Level = TraceLevel.Info});
+
+            if (_pithosStatus == PithosStatus.InSynch)
+                StatusMessage = "All files up to date";
                }
 
 
@@ -838,7 +861,7 @@ namespace Pithos.Client.WPF {
                    var progress = notification as ProgressNotification;
                    if (progress != null)
                    {
-                       StatusMessage = String.Format("Pithos {0}\r\n{1} {2} of {3} - {4}",
+                       StatusMessage = String.Format("Pithos {0}\r\n{1} {2:p2} of {3} - {4}",
                                                      _fileVersion.Value.FileVersion, 
                                               progress.Action,
                                                      progress.Block/(double)progress.TotalBlocks,
@@ -847,41 +870,60 @@ namespace Pithos.Client.WPF {
                        return;
                    }
 
+                   var info = notification as StatusNotification;
+            if (info != null)
+            {
+                StatusMessage = String.Format("Pithos {0}\r\n{1}",
+                                              _fileVersion.Value.FileVersion,
+                                              info.Title);
+                return;
+            }
                        if (String.IsNullOrWhiteSpace(notification.Message) && String.IsNullOrWhiteSpace(notification.Title))
                                return;
 
-                       BalloonIcon icon;
-                       switch (notification.Level)
-                       {
-                               case TraceLevel.Error:
-                                       icon = BalloonIcon.Error;
-                                       break;
-                               case TraceLevel.Info:
-                               case TraceLevel.Verbose:
-                                       icon = BalloonIcon.Info;
-                                       break;
-                               case TraceLevel.Warning:
-                                       icon = BalloonIcon.Warning;
-                                       break;
-                               default:
-                                       icon = BalloonIcon.None;
-                                       break;
-                       }
-                       
-                       if (Settings.ShowDesktopNotifications)
-                       {
-                               var tv = (ShellView) GetView();
-                           System.Action clickAction = null;
-                if (notification is ExpirationNotification)
-                {
-                    clickAction = ()=>ShowPreferences("AccountTab");
-                }
-                               var balloon=new PithosBalloon{Title=notification.Title,Message=notification.Message,Icon=icon,ClickAction=clickAction};
-                               tv.TaskbarView.ShowCustomBalloon(balloon,PopupAnimation.Fade,4000);
-//                             tv.TaskbarView.ShowBalloonTip(notification.Title, notification.Message, icon);
-                       }
+                       ShowBalloonFor(notification);
                }
-               #endregion
+
+           private void ShowBalloonFor(Notification notification)
+           {
+            Contract.Requires(notification!=null);
+            
+            if (!Settings.ShowDesktopNotifications) 
+                return;
+            
+            BalloonIcon icon;
+               switch (notification.Level)
+               {
+                   case TraceLevel.Info:
+                   case TraceLevel.Verbose:
+                       return;
+                case TraceLevel.Error:
+                    icon = BalloonIcon.Error;
+                    break;
+                case TraceLevel.Warning:
+                       icon = BalloonIcon.Warning;
+                       break;
+                   default:
+                       return;
+               }
+
+               var tv = (ShellView) GetView();
+               System.Action clickAction = null;
+               if (notification is ExpirationNotification)
+               {
+                   clickAction = () => ShowPreferences("AccountTab");
+               }
+               var balloon = new PithosBalloon
+                                 {
+                                     Title = notification.Title,
+                                     Message = notification.Message,
+                                     Icon = icon,
+                                     ClickAction = clickAction
+                                 };
+               tv.TaskbarView.ShowCustomBalloon(balloon, PopupAnimation.Fade, 4000);
+           }
+
+           #endregion
 
                public void Handle(ShowFilePropertiesEvent message)
                {
index e445dde..99f3f17 100644 (file)
                        <appender-ref ref="TraceAppender"/>
                </logger>
 
+               <logger name="Pithos" additivity="false">
+                       <level value="DEBUG"/>
+                       <appender-ref ref="TraceAppender"/>
+               </logger>
+
                <root>
                        <level value="DEBUG" />
                        <appender-ref ref="LossyFileAppender" />
index 2834922..fe9b752 100644 (file)
@@ -6,6 +6,7 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading;
+using System.Threading.Tasks;
 using Pithos.Interfaces;
 using Pithos.Network;
 
@@ -146,6 +147,11 @@ namespace Pithos.Core.Test
             throw new NotImplementedException();
         }
 
+        public void EnsureFileState(string path)
+        {
+            throw new NotImplementedException();
+        }
+
 
         private PithosStatus _pithosStatus = PithosStatus.InSynch;
         public void SetPithosStatus(PithosStatus status)
@@ -158,9 +164,10 @@ namespace Pithos.Core.Test
             return _pithosStatus;
         }
 
-        public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)
+        public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string shortHash = null)
         {
             _overlayCache[path] = overlayStatus;
+            return Task.Factory.StartNew(()=>{});
         }
 
         public void RemoveFileOverlayStatus(string path)
@@ -177,7 +184,7 @@ namespace Pithos.Core.Test
             _overlayCache.TryRemove(oldPath, out value);
         }
 
-        public void UpdateFileChecksum(string path, string checksum)
+        public void UpdateFileChecksum(string path, string shortHash, string checksum)
         {
             _checksums[path] = checksum;
         }
index 0956bfc..e749f33 100644 (file)
@@ -53,7 +53,8 @@ namespace Pithos.Core
     public class Agent<TMessage> : IDisposable
     {
         private readonly ConcurrentQueue<TMessage> _queue;
-        private readonly AsyncProducerConsumerCollection<TMessage> _messages;
+        //private readonly AsyncCollection<TMessage> _messages;
+        private readonly AsyncCollection<TMessage> _messages;
         private readonly CancellationTokenSource _cancelSource = new CancellationTokenSource();
         public CancellationToken CancellationToken;
 
@@ -63,7 +64,7 @@ namespace Pithos.Core
         public Agent(Action<Agent<TMessage>> action)
         {
             _queue=new ConcurrentQueue<TMessage>();
-            _messages = new AsyncProducerConsumerCollection<TMessage>(_queue);            
+            _messages = new AsyncCollection<TMessage>(_queue);            
             _process = action;
             CancellationToken = _cancelSource.Token;
         }
@@ -78,6 +79,16 @@ namespace Pithos.Core
             _messages.Add(message);
         }
 
+        ConcurrentDictionary<TMessage,TaskCompletionSource<object>> _awaiters=new ConcurrentDictionary<TMessage,TaskCompletionSource<object>>();
+
+        public Task PostAndAwait(TMessage message)
+        {            
+            var tcs = new TaskCompletionSource<object>();
+            _awaiters[message] = tcs;
+            Post(message);
+            return tcs.Task;
+        }
+
         /// <summary>
         /// Receives a message asynchronously, optionally with a timeout. Receive throws a TimeoutException if the timeout expires
         /// </summary>
@@ -87,6 +98,13 @@ namespace Pithos.Core
             return _messages.Take();
         }
 
+        public void NotifyComplete(TMessage message)
+        {
+            TaskCompletionSource<object> tcs;
+            if (_awaiters.TryRemove(message,out tcs))
+                tcs.SetResult(null);
+        }
+
 
 
         /// <summary>
@@ -148,7 +166,6 @@ namespace Pithos.Core
             if (disposing)
             {
                 Stop();
-                _messages.Dispose();
                 _cancelSource.Dispose();
             }
         }
@@ -209,7 +226,7 @@ namespace Pithos.Core
                         onError(ex);
                 }
                 return default(T);
-            },CancellationToken);
+            });
         }
     }
 }
diff --git a/trunk/Pithos.Core/Agents/AsyncCollection.cs b/trunk/Pithos.Core/Agents/AsyncCollection.cs
new file mode 100644 (file)
index 0000000..873a034
--- /dev/null
@@ -0,0 +1,63 @@
+//--------------------------------------------------------------------------
+// 
+//  Copyright (c) Microsoft Corporation.  All rights reserved. 
+// 
+//  File: AsyncProducerConsumerCollection.cs
+//
+//--------------------------------------------------------------------------
+
+using System;
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Pithos.Core
+{
+    /// <summary>Provides an asynchronous producer/consumer collection.</summary>
+    [DebuggerDisplay("Count={CurrentCount}")]
+    public sealed class AsyncCollection<T> 
+    {
+        /// <summary>Asynchronous semaphore used to keep track of asynchronous work.</summary>
+        private AsyncSemaphore _semaphore = new AsyncSemaphore(0);
+        /// <summary>The data stored in the collection.</summary>
+        private IProducerConsumerCollection<T> _collection;
+
+        /// <summary>Initializes the asynchronous producer/consumer collection to store data in a first-in-first-out (FIFO) order.</summary>
+        public AsyncCollection() : this(new ConcurrentQueue<T>()) { }
+
+        /// <summary>Initializes the asynchronous producer/consumer collection.</summary>
+        /// <param name="collection">The underlying collection to use to store data.</param>
+        public AsyncCollection(IProducerConsumerCollection<T> collection)
+        {
+            if (collection == null) throw new ArgumentNullException("collection");
+            _collection = collection;
+        }
+
+        /// <summary>Adds an element to the collection.</summary>
+        /// <param name="item">The item to be added.</param>
+        public void Add(T item)
+        {
+            if (_collection.TryAdd(item))
+                Task.Factory.StartNew(s => ((AsyncSemaphore)s).Release(),
+                    _semaphore, CancellationToken.None, TaskCreationOptions.PreferFairness, TaskScheduler.Default);
+            else throw new InvalidOperationException("Invalid collection");
+        }
+
+        /// <summary>Takes an element from the collection asynchronously.</summary>
+        /// <returns>A Task that represents the element removed from the collection.</returns>
+        public Task<T> Take()
+        {
+            return _semaphore.WaitAsync().ContinueWith(_ =>
+            {
+                T result;
+                if (!_collection.TryTake(out result)) throw new InvalidOperationException("Invalid collection");
+                return result;
+            }, TaskContinuationOptions.OnlyOnRanToCompletion);
+        }
+
+        /// <summary>Gets the number of elements in the collection.</summary>
+        public int Count { get { return _collection.Count; } }
+
+    }
+}
\ No newline at end of file
index 5ae6ea5..783a145 100644 (file)
@@ -87,8 +87,11 @@ namespace Pithos.Core.Agents
             while (true)\r
             {\r
                 var tcs = _tcs;\r
-                if (!tcs.Task.IsCompleted ||\r
-                    Interlocked.CompareExchange(ref _tcs, new TaskCompletionSource<bool>(), tcs) == tcs)\r
+                if (!tcs.Task.IsCompleted)\r
+                    return;\r
+#pragma warning disable 420\r
+                if (Interlocked.CompareExchange(ref _tcs, new TaskCompletionSource<bool>(), tcs) == tcs)\r
+#pragma warning restore 420\r
                     return;\r
             }\r
         }\r
diff --git a/trunk/Pithos.Core/Agents/AsyncSemaphore.cs b/trunk/Pithos.Core/Agents/AsyncSemaphore.cs
new file mode 100644 (file)
index 0000000..c22a4a8
--- /dev/null
@@ -0,0 +1,64 @@
+// -----------------------------------------------------------------------
+// <copyright file="AsyncSemaphore.cs" company="Microsoft">
+// TODO: Update copyright text.
+// </copyright>
+// -----------------------------------------------------------------------
+
+using System.Threading.Tasks;
+
+namespace Pithos.Core
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Text;
+
+    /// <summary>
+    /// TODO: Update summary.
+    /// </summary>
+    public class AsyncSemaphore
+    {
+        private readonly static Task s_completed = TaskEx.FromResult(true);
+        private readonly Queue<TaskCompletionSource<bool>> m_waiters = new Queue<TaskCompletionSource<bool>>();
+        private int m_currentCount; 
+
+        public AsyncSemaphore(int initialCount)
+        {
+            if (initialCount < 0) throw new ArgumentOutOfRangeException("initialCount");
+            m_currentCount = initialCount; 
+        }
+        public Task WaitAsync()
+        {
+
+            lock (m_waiters)
+            {
+                if (m_currentCount > 0)
+                {
+                    --m_currentCount;
+                    return s_completed;
+                }
+                else
+                {
+                    var waiter = new TaskCompletionSource<bool>();
+                    m_waiters.Enqueue(waiter);
+                    return waiter.Task;
+                }
+            }
+
+        }
+
+        public void Release()
+        {
+            TaskCompletionSource<bool> toRelease = null;
+            lock (m_waiters)
+            {
+                if (m_waiters.Count > 0)
+                    toRelease = m_waiters.Dequeue();
+                else
+                    ++m_currentCount;
+            }
+            if (toRelease != null)
+                toRelease.SetResult(true);
+        }
+    }
+}
index 2fb90ec..a58a3ea 100644 (file)
@@ -43,6 +43,7 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics.Contracts;
 using System.Linq;
+using System.Security.Cryptography;
 using System.Text;
 using System.IO;
 using System.Text.RegularExpressions;
@@ -87,5 +88,38 @@ namespace Pithos.Core.Agents
            return Signature.CalculateTreeHash(info.FullName, blockSize, algorithm).TopHash.ToHashString();
 
         }
+
+       /// <summary>
+       ///Calculates a simple hash for an entire file
+       /// </summary>
+       /// <param name="info">The file to hash</param>
+       /// <param name="hasher">The hash algorithm to use</param>
+       /// <returns>A hash value for the entire file. An empty string if the file does not exist.</returns>
+       public static string ComputeShortHash(this FileInfo info, HashAlgorithm hasher)
+       {
+           Contract.Requires(info != null);
+           Contract.Requires(hasher!= null);           
+
+           if (!info.Exists)
+               return String.Empty;
+
+           using (var stream = info.Open(FileMode.Open, FileAccess.Read, FileShare.Read))
+           {
+               var hash = hasher.ComputeHash(stream);
+               var hashString = hash.ToHashString();
+               return hashString;
+           }
+       }
+
+        public static string ComputeShortHash(this FileInfo info)
+       {
+           Contract.Requires(info != null);           
+
+           using (var hasher=HashAlgorithm.Create("sha1"))
+           {               
+               return ComputeShortHash(info,hasher);
+           }
+       }
+
     }
 }
index 8bd0bcc..3edc216 100644 (file)
@@ -207,7 +207,12 @@ namespace Pithos.Core.Agents
             if (File.Exists(FilePath))
                 File.Replace(TempPath, FilePath, null, true);
             else
+            {
+                var targetDirectory = Path.GetDirectoryName(FilePath);
+                if (!Directory.Exists(targetDirectory))
+                    Directory.CreateDirectory(targetDirectory);
                 File.Move(TempPath, FilePath);
+            }
         }
 
         private void ClearBlocks()
index 5163c3c..5576df1 100644 (file)
@@ -71,8 +71,8 @@ namespace Pithos.Core.Agents
         public readonly DateTime Created = DateTime.Now;
 
 
-        public Lazy<string> LocalHash { get; protected set; }
-        public Lazy<string> TopHash { get; set; }
+        public Lazy<TreeHash> TreeHash { get; protected set; }
+        //public Lazy<string> TopHash { get; set; }
 
 
         [ContractInvariantMethod]
@@ -102,12 +102,12 @@ namespace Pithos.Core.Agents
             LocalFile = localFile.WithProperCapitalization();
             CloudFile = cloudFile;
             FileState = state;
-            if (LocalFile != null)
-            {
+            
+            if (LocalFile == null) 
+                return;
 
-                LocalHash = new Lazy<string>(() => LocalFile.CalculateHash(blockSize,algorithm),
-                                             LazyThreadSafetyMode.ExecutionAndPublication);
-            }
+            TreeHash = new Lazy<TreeHash>(() => Signature.CalculateTreeHash(LocalFile, blockSize,algorithm),
+                                            LazyThreadSafetyMode.ExecutionAndPublication);
         }
 
         //Calculate the download path for the cloud file
@@ -239,7 +239,7 @@ namespace Pithos.Core.Agents
             OldCloudFile = CreateObjectInfoFor(accountInfo, oldFile);
 
             //This is a rename operation, a hash will not be used
-            LocalHash = new Lazy<string>(() => String.Empty, LazyThreadSafetyMode.ExecutionAndPublication);
+            TreeHash = new Lazy<TreeHash>(() => Network.TreeHash.Empty, LazyThreadSafetyMode.ExecutionAndPublication);
         }
 
         public override string ToString()
index 8cef5bf..cacb073 100644 (file)
 #endregion
 using System;
 using System.Collections.Generic;
-using System.ComponentModel.Composition;
-using System.Diagnostics;
 using System.Diagnostics.Contracts;
 using System.IO;
 using System.Linq;
 using System.Reflection;
-using System.Text;
 using System.Threading.Tasks;
 using Pithos.Interfaces;
 using Pithos.Network;
 using log4net;
-using log4net.Core;
 
 namespace Pithos.Core.Agents
 {
@@ -67,6 +63,8 @@ namespace Pithos.Core.Agents
 
         //[Import]
         public IStatusKeeper StatusKeeper { get; set; }
+
+        public IStatusNotification StatusNotification { get; set; }
         //[Import]
         public IPithosWorkflow Workflow { get; set; }
         //[Import]
@@ -89,13 +87,13 @@ namespace Pithos.Core.Agents
 
             AccountInfo = accountInfo;
             RootPath = rootPath;
-            _watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true};
+            _watcher = new FileSystemWatcher(rootPath) {IncludeSubdirectories = true,InternalBufferSize=8*4096};
             _adapter = new FileSystemWatcherAdapter(_watcher);
 
             _adapter.Changed += OnFileEvent;
             _adapter.Created += OnFileEvent;
             _adapter.Deleted += OnFileEvent;
-            _adapter.Renamed += OnRenameEvent;
+            //_adapter.Renamed += OnRenameEvent;
             _adapter.Moved += OnMoveEvent;
             _watcher.EnableRaisingEvents = true;
 
@@ -135,6 +133,8 @@ namespace Pithos.Core.Agents
 
             try
             {
+                //StatusKeeper.EnsureFileState(state.Path);
+                
                 UpdateFileStatus(state);
                 UpdateOverlayStatus(state);
                 UpdateFileChecksum(state);
@@ -193,10 +193,6 @@ namespace Pithos.Core.Agents
         {
             if (_watcher != null)
             {
-                _watcher.Changed -= OnFileEvent;
-                _watcher.Created -= OnFileEvent;
-                _watcher.Deleted -= OnFileEvent;
-                _watcher.Renamed -= OnRenameEvent;
                 _watcher.Dispose();
             }
             _watcher = null;
@@ -329,6 +325,7 @@ namespace Pithos.Core.Agents
         }
 
 
+/*
         //Post a Change message for renames containing the old and new names
         void OnRenameEvent(object sender, RenamedEventArgs e)
         {
@@ -347,8 +344,9 @@ namespace Pithos.Core.Agents
                 TriggeringChange = e.ChangeType
             });
         }
+*/
 
-        //Post a Change message for renames containing the old and new names
+        //Post a Change message for moves containing the old and new names
         void OnMoveEvent(object sender, MovedEventArgs e)
         {
             var oldFullPath = e.OldFullPath;
@@ -415,18 +413,20 @@ namespace Pithos.Core.Agents
             switch (state.Status)
             {
                 case FileStatus.Created:
+                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash);
+                    break;
                 case FileStatus.Modified:
-                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
+                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash);
                     break;
                 case FileStatus.Deleted:
                     //this.StatusAgent.RemoveFileOverlayStatus(state.Path);
                     break;
                 case FileStatus.Renamed:
                     this.StatusKeeper.ClearFileStatus(state.OldPath);
-                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified);
+                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Modified,state.ShortHash);
                     break;
                 case FileStatus.Unchanged:
-                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal);
+                    this.StatusKeeper.SetFileOverlayStatus(state.Path, FileOverlayStatus.Normal,state.ShortHash);
                     break;
             }
 
@@ -451,11 +451,16 @@ namespace Pithos.Core.Agents
             if (Directory.Exists(path))
                 return state;
 
+
             var info = new FileInfo(path);
-            string hash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
-            StatusKeeper.UpdateFileChecksum(path, hash);
+            StatusNotification.Notify(new StatusNotification(String.Format("Hashing [{0}]",info.Name)));
+
+            var shortHash = info.ComputeShortHash(); 
+            
+            string merkleHash = info.CalculateHash(StatusKeeper.BlockSize,StatusKeeper.BlockHash);
+            StatusKeeper.UpdateFileChecksum(path,shortHash, merkleHash);
 
-            state.Hash = hash;
+            state.Hash = merkleHash;
             return state;
         }
 
index 0dd40cd..2b69050 100644 (file)
 #endregion\r
 using System.Diagnostics.Contracts;\r
 using System.IO;\r
+using System.Reflection;\r
 using System.Threading.Tasks;\r
 using Pithos.Interfaces;\r
+using log4net;\r
 \r
 namespace Pithos.Core.Agents\r
 {\r
@@ -53,10 +55,13 @@ namespace Pithos.Core.Agents
     /// </summary>\r
     public class FileSystemWatcherAdapter\r
     {\r
-         public event FileSystemEventHandler Changed;\r
+        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);\r
+\r
+\r
+        public event FileSystemEventHandler Changed;\r
         public event FileSystemEventHandler Created;\r
         public event FileSystemEventHandler Deleted;\r
-        public event RenamedEventHandler Renamed;\r
+        //public event RenamedEventHandler Renamed;\r
         public event MovedEventHandler Moved;\r
 \r
         public FileSystemWatcherAdapter(FileSystemWatcher watcher)\r
@@ -68,8 +73,14 @@ namespace Pithos.Core.Agents
             watcher.Changed += OnChangeOrCreate;\r
             watcher.Created += OnChangeOrCreate;\r
             watcher.Deleted += OnDeleted;\r
-            watcher.Renamed += OnRename;            \r
-            \r
+            watcher.Renamed += OnRename;\r
+            watcher.Error += OnError;\r
+        }\r
+\r
+        private void OnError(object sender, ErrorEventArgs e)\r
+        {\r
+            var error = e.GetException();\r
+            Log.Error("FSW error",error);\r
         }\r
 \r
         private string _cachedDeletedFullPath;\r
@@ -86,7 +97,14 @@ namespace Pithos.Core.Agents
             Contract.Ensures(!String.IsNullOrWhiteSpace(_cachedDeletedFullPath));\r
             Contract.EndContractBlock();\r
 \r
-            //Handle any previously deleted event\r
+            TaskEx.Run(() => InnerOnDeleted(sender, e));\r
+        }\r
+\r
+        private void InnerOnDeleted(object sender, FileSystemEventArgs e)\r
+        {\r
+//Handle any previously deleted event\r
+            if (Log.IsDebugEnabled)\r
+                Log.DebugFormat("[{0}] for [{1}]", Enum.GetName(typeof(WatcherChangeTypes), e.ChangeType), e.FullPath);\r
             PropagateCachedDeleted(sender);\r
 \r
             //A delete event may be an actual delete event or the first event in a move action.\r
@@ -99,17 +117,17 @@ namespace Pithos.Core.Agents
             //      as this is actually a MOVE operation\r
             //Deleting by Shift+Delete results in a delete event for each file followed by the delete of the folder itself\r
             _cachedDeletedFullPath = e.FullPath;\r
-            \r
+\r
             //TODO: This requires synchronization of the _cachedDeletedFullPath field\r
             //TODO: This creates a new task for each file even though we can cancel any existing tasks if a new event arrives\r
             //Maybe, use a timer instead of a task\r
-            \r
+\r
             TaskEx.Delay(PropagateDelay).ContinueWith(t =>\r
-                                                           {\r
-                                                               var myPath = e.FullPath;\r
-                                                               if (_cachedDeletedFullPath==myPath)\r
-                                                                    PropagateCachedDeleted(sender);\r
-                                                           });\r
+                                                          {\r
+                                                              var myPath = e.FullPath;\r
+                                                              if (_cachedDeletedFullPath == myPath)\r
+                                                                  PropagateCachedDeleted(sender);\r
+                                                          });\r
         }\r
 \r
         private void OnRename(object sender, RenamedEventArgs e)\r
@@ -119,18 +137,56 @@ namespace Pithos.Core.Agents
             Contract.Ensures(_cachedDeletedFullPath == null);\r
             Contract.EndContractBlock();\r
 \r
+            TaskEx.Run(() => InnerRename(sender, e));\r
+        }\r
+\r
+        private void InnerRename(object sender, RenamedEventArgs e)\r
+        {\r
             try\r
             {\r
+                if (Log.IsDebugEnabled)\r
+                    Log.DebugFormat("[{0}] for [{1}]", Enum.GetName(typeof(WatcherChangeTypes), e.ChangeType), e.FullPath);\r
                 //Propagate any previous cached delete event\r
                 PropagateCachedDeleted(sender);\r
-                \r
-                if (Renamed!=null)\r
-                    Renamed(sender, e);\r
-                \r
+\r
+                if (Moved!= null)\r
+                {\r
+                    try\r
+                    {\r
+\r
+                        Moved(sender, new MovedEventArgs(e.FullPath,e.Name,e.OldFullPath,e.OldName));\r
+                    }\r
+                    catch (Exception exc)\r
+                    {\r
+                        Log.Error("Rename event error", exc);\r
+                        throw;\r
+                    }\r
+\r
+                    var directory = new DirectoryInfo(e.FullPath);\r
+                    if (directory.Exists)\r
+                    {\r
+                        var newDirectory = e.FullPath;\r
+                        var oldDirectory = e.OldFullPath;\r
+\r
+                        foreach (\r
+                            var child in\r
+                                directory.EnumerateFileSystemInfos("*", SearchOption.AllDirectories))\r
+                        {\r
+                            var newChildDirectory = Path.GetDirectoryName(child.FullName);\r
+\r
+                            var relativePath = child.AsRelativeTo(newDirectory);\r
+                            var relativeFolder = Path.GetDirectoryName(relativePath);\r
+                            var oldChildDirectory = Path.Combine(oldDirectory, relativeFolder);\r
+                            Moved(sender,\r
+                                  new MovedEventArgs(newChildDirectory, child.Name, oldChildDirectory,\r
+                                                     child.Name));\r
+                        }\r
+                    }\r
+                }\r
             }\r
             finally\r
             {\r
-                _cachedDeletedFullPath = null;    \r
+                _cachedDeletedFullPath = null;\r
             }\r
         }\r
 \r
@@ -142,9 +198,16 @@ namespace Pithos.Core.Agents
                 throw new ArgumentException("e");\r
             Contract.Ensures(_cachedDeletedFullPath == null);\r
             Contract.EndContractBlock();\r
+            TaskEx.Run(() => InnerChangeOrCreated(sender, e));\r
 \r
+        }\r
+\r
+        private void InnerChangeOrCreated(object sender, FileSystemEventArgs e)\r
+        {\r
             try\r
             {\r
+                if (Log.IsDebugEnabled)\r
+                    Log.DebugFormat("[{0}] for [{1}]",Enum.GetName(typeof(WatcherChangeTypes),e.ChangeType),e.FullPath);\r
                 //A Move action results in a sequence of a Delete and a Create or Change event\r
                 //If the actual action is a Move, raise a Move event instead of the actual event\r
                 if (HandleMoved(sender, e))\r
@@ -159,7 +222,7 @@ namespace Pithos.Core.Agents
                 {\r
                     actualEvent(sender, e);\r
                     //For Folders, raise Created events for all children\r
-                    RaiseCreatedForChildren(sender,e);\r
+                    RaiseCreatedForChildren(sender, e);\r
                 }\r
             }\r
             finally\r
@@ -167,7 +230,6 @@ namespace Pithos.Core.Agents
                 //Finally, make sure the cached path is cleared\r
                 _cachedDeletedFullPath = null;\r
             }\r
-\r
         }\r
 \r
         private void RaiseCreatedForChildren(object sender, FileSystemEventArgs e)\r
@@ -181,11 +243,20 @@ namespace Pithos.Core.Agents
             //Skip if this is not a folder\r
             if (!dir.Exists)\r
                 return;\r
-            foreach (var info in dir.EnumerateFileSystemInfos("*",SearchOption.AllDirectories))\r
+            try\r
             {\r
-                var path = Path.GetDirectoryName(info.FullName);\r
-                Created(sender,new FileSystemEventArgs(WatcherChangeTypes.Created,path,info.Name));\r
-            }            \r
+                foreach (var info in dir.EnumerateFileSystemInfos("*",SearchOption.AllDirectories))\r
+                {\r
+                    var path = Path.GetDirectoryName(info.FullName);\r
+                    Created(sender,new FileSystemEventArgs(WatcherChangeTypes.Created,path,info.Name));\r
+                }\r
+            }\r
+            catch (IOException exc)\r
+            {\r
+                TaskEx.Delay(1000)\r
+                    .ContinueWith(_=>RaiseCreatedForChildren(sender,e));\r
+                \r
+            }\r
         }\r
 \r
         private bool HandleMoved(object sender, FileSystemEventArgs e)\r
@@ -212,6 +283,9 @@ namespace Pithos.Core.Agents
 \r
             try\r
             {\r
+                if (Log.IsDebugEnabled)\r
+                    Log.DebugFormat("Moved for [{0}]",  e.FullPath);\r
+\r
                 //If the actual action is a Move, raise a Move event instead of the actual event\r
                 var newDirectory = Path.GetDirectoryName(e.FullPath);\r
                 var oldDirectory = Path.GetDirectoryName(_cachedDeletedFullPath);\r
@@ -261,6 +335,9 @@ namespace Pithos.Core.Agents
             var deletedFileName = Path.GetFileName(_cachedDeletedFullPath);\r
             var deletedFileDirectory = Path.GetDirectoryName(_cachedDeletedFullPath);\r
 \r
+            if (Log.IsDebugEnabled)\r
+                Log.DebugFormat("Propagating delete for [{0}]", _cachedDeletedFullPath);\r
+\r
             //Only a single file Delete event is raised when moving a file to the Recycle Bin, as this is actually a MOVE operation\r
             //In this case we need to raise the proper events for all child objects of the deleted directory.\r
             //UNFORTUNATELY, this can't be detected here, eg. by retrieving the child objects, because they are already deleted\r
index 47c4c37..97e5b57 100644 (file)
@@ -89,6 +89,12 @@ namespace Pithos.Core.Agents
 
         public void Start()
         {
+            if (_agent != null)
+                return;
+
+            if (Log.IsDebugEnabled)
+                Log.Debug("Starting Network Agent");
+
             _agent = Agent<CloudAction>.Start(inbox =>
             {
                 Action loop = null;
@@ -123,6 +129,7 @@ namespace Pithos.Core.Agents
 
                 try
                 {
+                    StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Processing");
                     _proceedEvent.Reset();
                     //UpdateStatus(PithosStatus.Syncing);
                     var accountInfo = action.AccountInfo;
@@ -204,7 +211,7 @@ namespace Pithos.Core.Agents
                 {
                     if (_agent.IsEmpty)
                         _proceedEvent.Set();
-                    UpdateStatus(PithosStatus.InSynch);                                        
+                    UpdateStatus(PithosStatus.LocalComplete);                                        
                 }
             }
         }
@@ -212,8 +219,8 @@ namespace Pithos.Core.Agents
 
         private void UpdateStatus(PithosStatus status)
         {
-            StatusKeeper.SetPithosStatus(status);
-            StatusNotification.Notify(new Notification());
+            StatusNotification.SetPithosStatus(status);
+            //StatusNotification.Notify(new Notification());
         }
 
         private void RenameLocalFile(AccountInfo accountInfo, CloudAction action)
@@ -297,17 +304,16 @@ namespace Pithos.Core.Agents
 
                 var cloudHash = cloudFile.Hash.ToLower();
                 var previousCloudHash = cloudFile.PreviousHash.ToLower();
-                var localHash = action.LocalHash.Value.ToLower();
-                var topHash = action.TopHash.Value.ToLower();
+                var localHash = action.TreeHash.Value.TopHash.ToHashString();// LocalHash.Value.ToLower();
+                //var topHash = action.TopHash.Value.ToLower();
 
                 //At this point we know that an object has changed on the server and that a local
                 //file already exists. We need to decide whether the file has only changed on 
                 //the server or there is a conflicting change on the client.
                 //
 
-                //Not enough to compare only the local hashes (MD5), also have to compare the tophashes            
-                //If any of the hashes match, we are done
-                if ((cloudHash == localHash || cloudHash == topHash))
+                //If the hashes match, we are done
+                if (cloudHash == localHash)
                 {
                     Log.InfoFormat("Skipping {0}, hashes match", downloadPath);
                     return;
@@ -351,12 +357,14 @@ namespace Pithos.Core.Agents
             Contract.EndContractBlock();
 
             _deleteAgent.ProceedEvent.Wait();
+/*
 
             //If the action targets a local file, add a treehash calculation
             if (!(cloudAction is CloudDeleteAction) && cloudAction.LocalFile as FileInfo != null)
             {
                 var accountInfo = cloudAction.AccountInfo;
                 var localFile = (FileInfo) cloudAction.LocalFile;
+
                 if (localFile.Length > accountInfo.BlockSize)
                     cloudAction.TopHash =
                         new Lazy<string>(() => Signature.CalculateTreeHashAsync(localFile,
@@ -367,12 +375,14 @@ namespace Pithos.Core.Agents
                 {
                     cloudAction.TopHash = new Lazy<string>(() => cloudAction.LocalHash.Value);
                 }
+
             }
             else
             {
                 //The hash for a directory is the empty string
                 cloudAction.TopHash = new Lazy<string>(() => String.Empty);
             }
+*/
             
             if (cloudAction is CloudDeleteAction)
                 _deleteAgent.Post((CloudDeleteAction)cloudAction);
@@ -497,22 +507,28 @@ namespace Pithos.Core.Agents
                     }
                     else
                     {
-                        //Retrieve the hashmap from the server
-                        var serverHash = await client.GetHashMap(account, container, url);
-                        //If it's a small file
-                        if (serverHash.Hashes.Count == 1)
-                            //Download it in one go
-                            await
-                                DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath,
-                                                        serverHash);
-                            //Otherwise download it block by block
-                        else
-                            await DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath, serverHash);
-
-                        if (cloudFile.AllowedTo == "read")
+                        var isChanged = IsObjectChanged(cloudFile, localPath);
+                        if (isChanged)
                         {
-                            var attributes = File.GetAttributes(localPath);
-                            File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly);
+                            //Retrieve the hashmap from the server
+                            var serverHash = await client.GetHashMap(account, container, url);
+                            //If it's a small file
+                            if (serverHash.Hashes.Count == 1)
+                                //Download it in one go
+                                await
+                                    DownloadEntireFileAsync(accountInfo, client, cloudFile, relativeUrl, localPath,
+                                                            serverHash);
+                                //Otherwise download it block by block
+                            else
+                                await
+                                    DownloadWithBlocks(accountInfo, client, cloudFile, relativeUrl, localPath,
+                                                       serverHash);
+
+                            if (cloudFile.AllowedTo == "read")
+                            {
+                                var attributes = File.GetAttributes(localPath);
+                                File.SetAttributes(localPath, attributes | FileAttributes.ReadOnly);
+                            }
                         }
                     }
 
@@ -523,6 +539,28 @@ namespace Pithos.Core.Agents
             }
         }
 
+        private bool IsObjectChanged(ObjectInfo cloudFile, string localPath)
+        {
+            //If the target is a directory, there are no changes to download
+            if (Directory.Exists(localPath))
+                return false;
+            //If the file doesn't exist, we have a chagne
+            if (!File.Exists(localPath)) 
+                return true;
+            //If there is no stored state, we have a change
+            var localState = StatusKeeper.GetStateByFilePath(localPath);
+            if (localState == null)
+                return true;
+
+            var info = new FileInfo(localPath);
+            var shortHash = info.ComputeShortHash();
+            //If the file is different from the stored state, we have a change
+            if (localState.ShortHash != shortHash)
+                return true;
+            //If the top hashes differ, we have a change
+            return (localState.Checksum != cloudFile.Hash);
+        }
+
         //Download a small file with a single GET operation
         private async Task DownloadEntireFileAsync(AccountInfo accountInfo, CloudFilesClient client, ObjectInfo cloudFile, Uri relativeUrl, string filePath,TreeHash serverHash)
         {
@@ -539,20 +577,7 @@ namespace Pithos.Core.Agents
             Contract.EndContractBlock();
 
             var localPath = Pithos.Interfaces.FileInfoExtensions.GetProperFilePathCapitalization(filePath);
-            //If the file already exists
-            if (File.Exists(localPath))
-            {
-                //First check with MD5 as this is a small file
-                var localMD5 = Signature.CalculateMD5(localPath);
-                var cloudHash=serverHash.TopHash.ToHashString();
-                if (localMD5==cloudHash)
-                    return;
-                //Then check with a treehash
-                var localTreeHash = Signature.CalculateTreeHash(localPath, serverHash.BlockSize, serverHash.BlockHash);
-                var localHash = localTreeHash.TopHash.ToHashString();
-                if (localHash==cloudHash)
-                    return;
-            }
+            StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Downloading {0}",Path.GetFileName(localPath)));
             StatusNotification.Notify(new CloudNotification { Data = cloudFile });
 
             var fileAgent = GetFileAgent(accountInfo);
@@ -608,7 +633,7 @@ namespace Pithos.Core.Agents
             var blockUpdater = new BlockUpdater(fileAgent.CachePath, localPath, relativePath, serverHash);
 
             
-                        
+            StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,String.Format("Calculating hashmap for {0} before download",Path.GetFileName(localPath)));
             //Calculate the file's treehash
             var treeHash = await Signature.CalculateTreeHashAsync(localPath, serverHash.BlockSize, serverHash.BlockHash, 2);
                 
@@ -676,6 +701,10 @@ namespace Pithos.Core.Agents
 
                     if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))
                         return;
+                    
+                    //
+                    if (action.FileState == null)
+                        action.FileState = StatusKeeper.GetStateByFilePath(fileInfo.FullName);
                     //Do not upload files in conflict
                     if (action.FileState.FileStatus == FileStatus.Conflict )
                     {
@@ -724,17 +753,17 @@ namespace Pithos.Core.Agents
 
                         var client = new CloudFilesClient(accountInfo);
                         //Even if GetObjectInfo times out, we can proceed with the upload            
-                        var info = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
+                        var cloudInfo = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
 
                         //If this is a read-only file, do not upload changes
-                        if (info.AllowedTo == "read")
+                        if (cloudInfo.AllowedTo == "read")
                             return;
 
                         //TODO: Check how a directory hash is calculated -> All dirs seem to have the same hash
                             if (fileInfo is DirectoryInfo)
                             {
                                 //If the directory doesn't exist the Hash property will be empty
-                                if (String.IsNullOrWhiteSpace(info.Hash))
+                                if (String.IsNullOrWhiteSpace(cloudInfo.Hash))
                                     //Go on and create the directory
                                     await
                                         client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName,
@@ -743,16 +772,28 @@ namespace Pithos.Core.Agents
                             else
                             {
 
-                                var cloudHash = info.Hash.ToLower();
+                                var cloudHash = cloudInfo.Hash.ToLower();
+
+                                StatusNotification.Notify(new StatusNotification(String.Format("Hashing {0} for Upload",fileInfo.Name)));
+
+                                //TODO: This is the same as the calculation for the Local Hash!
+                                //First, calculate the tree hash
+/*
+                                var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize,
+                                                                                      accountInfo.BlockHash, 2);
+*/
 
-                                var hash = action.LocalHash.Value;
-                                var topHash = action.TopHash.Value;
+
+                                var treeHash = action.TreeHash.Value;
+                                var topHash = treeHash.TopHash.ToHashString();
+                                
+                                //var topHash = action.TopHash.Value;
 
                                 //If the file hashes match, abort the upload
-                                if (hash == cloudHash || topHash == cloudHash)
+                                if (topHash == cloudHash /*|| topHash == cloudHash*/)
                                 {
                                     //but store any metadata changes 
-                                    StatusKeeper.StoreInfo(fullFileName, info);
+                                    StatusKeeper.StoreInfo(fullFileName, cloudInfo);
                                     Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
                                     return;
                                 }
@@ -765,10 +806,6 @@ namespace Pithos.Core.Agents
                                 //Upload even small files using the Hashmap. The server may already contain
                                 //the relevant block
 
-                                //First, calculate the tree hash
-                                var treeHash = await Signature.CalculateTreeHashAsync(fullFileName, accountInfo.BlockSize,
-                                                                                      accountInfo.BlockHash, 2);
-
                                 //TODO: If the upload fails with a 403, abort it and mark conflict
 
                                 await
@@ -847,6 +884,8 @@ namespace Pithos.Core.Agents
                 throw new ArgumentException("Invalid container","cloudFile");
             Contract.EndContractBlock();
 
+            StatusNotification.Notify(new StatusNotification(String.Format("Uploading {0} for Upload", fileInfo.Name)));
+
             var fullFileName = fileInfo.GetProperCapitalization();
 
             var account = cloudFile.Account ?? accountInfo.UserName;
@@ -886,16 +925,22 @@ namespace Pithos.Core.Agents
                 //Repeat until there are no more missing hashes                
                 missingHashes = await client.PutHashMap(account, container, url, treeHash);
             }
+
             ReportUploadProgress(fileInfo.Name, missingHashes.Count, missingHashes.Count, fileInfo.Length);
         }
 
         private void ReportUploadProgress(string fileName,int block, int totalBlocks, long fileSize)
         {
-            StatusNotification.Notify(new ProgressNotification(fileName,"Uploading",block,totalBlocks,fileSize) );            
+            StatusNotification.Notify(totalBlocks == 0
+                                          ? new ProgressNotification(fileName, "Uploading", 1, 1, fileSize)
+                                          : new ProgressNotification(fileName, "Uploading", block, totalBlocks, fileSize));
         }
+
         private void ReportDownloadProgress(string fileName,int block, int totalBlocks, long fileSize)
         {
-            StatusNotification.Notify(new ProgressNotification(fileName,"Downloading",block,totalBlocks,fileSize) );            
+            StatusNotification.Notify(totalBlocks == 0
+                                          ? new ProgressNotification(fileName, "Downloading", 1, 1, fileSize)
+                                          : new ProgressNotification(fileName, "Downloading", block, totalBlocks, fileSize));
         }
     }
 
index 2089da7..23b39b7 100644 (file)
@@ -105,10 +105,11 @@ namespace Pithos.Core.Agents
         /// <returns></returns>\r
         public async Task PollRemoteFiles(DateTime? since = null)\r
         {\r
-            Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!");\r
+            if (Log.IsDebugEnabled)\r
+                Log.DebugFormat("Polling changes after [{0}]",since);\r
 \r
-            UpdateStatus(PithosStatus.Syncing);\r
-            StatusNotification.Notify(new PollNotification());\r
+            Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!");\r
+            \r
 \r
             using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
             {\r
@@ -116,6 +117,8 @@ namespace Pithos.Core.Agents
                 var nextSince = since;\r
                 try\r
                 {\r
+                    UpdateStatus(PithosStatus.PollSyncing);\r
+\r
                     //Next time we will check for all changes since the current check minus 1 second\r
                     //This is done to ensure there are no discrepancies due to clock differences\r
                     var current = DateTime.Now.AddSeconds(-1);\r
@@ -135,7 +138,7 @@ namespace Pithos.Core.Agents
                     //In case of failure retry with the same "since" value\r
                 }\r
 \r
-                UpdateStatus(PithosStatus.InSynch);\r
+                UpdateStatus(PithosStatus.PollComplete);\r
                 //The multiple try blocks are required because we can't have an await call\r
                 //inside a finally block\r
                 //TODO: Find a more elegant solution for reschedulling in the event of an exception\r
@@ -512,8 +515,8 @@ namespace Pithos.Core.Agents
         {\r
             try\r
             {\r
-                StatusKeeper.SetPithosStatus(status);\r
-                StatusNotification.Notify(new Notification());\r
+                StatusNotification.SetPithosStatus(status);\r
+                //StatusNotification.Notify(new Notification());\r
             }\r
             catch (Exception exc)\r
             {\r
index e9c6e8c..8efda7b 100644 (file)
@@ -48,10 +48,12 @@ using System.Diagnostics.Contracts;
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Security.Cryptography;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Castle.ActiveRecord;
+using Castle.ActiveRecord.Framework;
 using Castle.ActiveRecord.Framework.Config;
 using Pithos.Interfaces;
 using Pithos.Network;
@@ -185,6 +187,7 @@ namespace Pithos.Core.Agents
                         {
                             Log.ErrorFormat("[ERROR] STATE \n{0}", ex);
                         }
+                        queue.NotifyComplete(action);
 // ReSharper disable AccessToModifiedClosure
                         queue.DoAsync(loop);
 // ReSharper restore AccessToModifiedClosure
@@ -201,7 +204,7 @@ namespace Pithos.Core.Agents
         {
             _persistenceAgent.Stop();            
         }
-       
+               
 
         public void ProcessExistingFiles(IEnumerable<FileInfo> existingFiles)
         {
@@ -230,44 +233,52 @@ namespace Pithos.Core.Agents
                         where missingStates.Contains(state.Id)
                         select new { File = default(FileInfo), State = state };
 
-            var pairs = currentFiles.Union(deletedFiles);
+            var pairs = currentFiles.Union(deletedFiles).ToList();
 
-            foreach(var pair in pairs)
+            using (var shortHasher = HashAlgorithm.Create("sha1"))
             {
-                var fileState = pair.State;
-                var file = pair.File;
-                if (fileState == null)
-                {
-                    //This is a new file
-                    var fullPath = pair.File.FullName;
-                    var createState = FileState.CreateForAsync(fullPath, BlockSize, BlockHash);
-                    createState.ContinueWith(state => _persistenceAgent.Post(state.Result.Create));
-                }                
-                else if (file == null)
+                foreach (var pair in pairs)
                 {
-                    //This file was deleted while we were down. We should mark it as deleted
-                    //We have to go through UpdateStatus here because the state object we are using
-                    //was created by a different ORM session.
-                    _persistenceAgent.Post(()=> UpdateStatusDirect(fileState.Id, FileStatus.Deleted));                    
-                }
-                else
-                {
-                    //This file has a matching state. Need to check for possible changes
-                    var hashString = file.CalculateHash(BlockSize,BlockHash);
-                    //TODO: Need a way to attach the hashes to the filestate so we don't
-                    //recalculate them each time a call to calculate has is made
-                    //We can either store them to the filestate or add them to a 
-                    //dictionary
-
-                    //If the hashes don't match the file was changed
-                    if (fileState.Checksum != hashString)
+                    var fileState = pair.State;
+                    var file = pair.File;
+                    if (fileState == null)
                     {
-                        _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Modified));
-                    }                    
+                        //This is a new file                        
+                        var createState = FileState.CreateFor(file);
+                        _persistenceAgent.Post(createState.Create);                        
+                    }
+                    else if (file == null)
+                    {
+                        //This file was deleted while we were down. We should mark it as deleted
+                        //We have to go through UpdateStatus here because the state object we are using
+                        //was created by a different ORM session.
+                        _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Deleted));
+                    }
+                    else
+                    {
+                        //This file has a matching state. Need to check for possible changes
+                        //To check for changes, we use the cheap (in CPU terms) SHA1 algorithm
+                        //on the entire file.
+                        
+                        var hashString = file.ComputeShortHash(shortHasher);                        
+                        //TODO: Need a way to attach the hashes to the filestate so we don't
+                        //recalculate them each time a call to calculate has is made
+                        //We can either store them to the filestate or add them to a 
+                        //dictionary
+
+                        //If the hashes don't match the file was changed
+                        if (fileState.ShortHash != hashString)
+                        {
+                            _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Modified));
+                        }
+                    }
                 }
-            };            
+            }
+                        
          
         }
+        
+
 
         private int UpdateStatusDirect(Guid id, FileStatus status)
         {
@@ -387,17 +398,6 @@ namespace Pithos.Core.Agents
 
         }
 
-        private PithosStatus _pithosStatus=PithosStatus.InSynch;       
-
-        public void SetPithosStatus(PithosStatus status)
-        {
-            _pithosStatus = status;
-        }
-
-        public PithosStatus GetPithosStatus()
-        {
-            return _pithosStatus;
-        }
 
 
         private readonly string _pithosDataPath;
@@ -415,7 +415,7 @@ namespace Pithos.Core.Agents
             {
                 
                 using (var connection = GetConnection())
-                using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum   ,Version    ,VersionTimeStamp,IsShared   ,SharedBy   ,ShareWrite  from FileState where FilePath=:path COLLATE NOCASE", connection))
+                using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,ShortHash,Version    ,VersionTimeStamp,IsShared   ,SharedBy   ,ShareWrite  from FileState where FilePath=:path COLLATE NOCASE", connection))
                 {
                     
                     command.Parameters.AddWithValue("path", path);
@@ -433,11 +433,12 @@ namespace Pithos.Core.Agents
                                                 OverlayStatus =reader.IsDBNull(2)?FileOverlayStatus.Unversioned: (FileOverlayStatus) reader.GetInt64(2),
                                                 FileStatus = reader.IsDBNull(3)?FileStatus.Missing:(FileStatus) reader.GetInt64(3),
                                                 Checksum = reader.IsDBNull(4)?"":reader.GetString(4),
-                                                Version = reader.IsDBNull(5)?default(long):reader.GetInt64(5),
-                                                VersionTimeStamp = reader.IsDBNull(6)?default(DateTime):reader.GetDateTime(6),
-                                                IsShared = !reader.IsDBNull(7) && reader.GetBoolean(7),
-                                                SharedBy = reader.IsDBNull(8)?"":reader.GetString(8),
-                                                ShareWrite = !reader.IsDBNull(9) && reader.GetBoolean(9)
+                                                ShortHash= reader.IsDBNull(5)?"":reader.GetString(5),
+                                                Version = reader.IsDBNull(6)?default(long):reader.GetInt64(6),
+                                                VersionTimeStamp = reader.IsDBNull(7)?default(DateTime):reader.GetDateTime(7),
+                                                IsShared = !reader.IsDBNull(8) && reader.GetBoolean(8),
+                                                SharedBy = reader.IsDBNull(9)?"":reader.GetString(9),
+                                                ShareWrite = !reader.IsDBNull(10) && reader.GetBoolean(10)
                                             };
 /*
                             var state = new FileState
@@ -518,7 +519,7 @@ namespace Pithos.Core.Agents
             return connection;
         }
 
-        public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)
+       /* public void SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus)
         {
             if (String.IsNullOrWhiteSpace(path))
                 throw new ArgumentNullException("path");
@@ -527,6 +528,17 @@ namespace Pithos.Core.Agents
             Contract.EndContractBlock();
 
             _persistenceAgent.Post(() => FileState.StoreOverlayStatus(path,overlayStatus));
+        }*/
+
+        public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string shortHash = null)
+        {
+            if (String.IsNullOrWhiteSpace(path))
+                throw new ArgumentNullException("path");
+            if (!Path.IsPathRooted(path))
+                throw new ArgumentException("The path must be rooted","path");
+            Contract.EndContractBlock();
+
+            return _persistenceAgent.PostAndAwait(() => FileState.StoreOverlayStatus(path,overlayStatus,shortHash));
         }
 
        /* public void RenameFileOverlayStatus(string oldPath, string newPath)
@@ -627,12 +639,13 @@ namespace Pithos.Core.Agents
                     else
                     {
                         command.CommandText =
-                            "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,FileStatus,OverlayStatus) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:fileStatus,:overlayStatus)";
+                            "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,ShortHash,FileStatus,OverlayStatus) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:shortHash,:fileStatus,:overlayStatus)";
                         command.Parameters.AddWithValue("id", Guid.NewGuid());
                     }
 
                     command.Parameters.AddWithValue("path", path);
                     command.Parameters.AddWithValue("checksum", objectInfo.Hash);
+                    command.Parameters.AddWithValue("shortHash", "");
                     command.Parameters.AddWithValue("version", objectInfo.Version);
                     command.Parameters.AddWithValue("versionTimeStamp",
                                                     objectInfo.VersionTimestamp);
@@ -736,6 +749,21 @@ namespace Pithos.Core.Agents
             return children;
         }
 
+        public void EnsureFileState(string path)
+        {
+            var existingState = GetStateByFilePath(path);
+            if (existingState != null)
+                return;
+            var fileInfo = FileInfoExtensions.FromPath(path);
+            using (new SessionScope())
+            {
+                var newState = FileState.CreateFor(fileInfo);
+                newState.FileStatus=FileStatus.Missing;
+                _persistenceAgent.PostAndAwait(newState.CreateAndFlush).Wait();
+            }
+
+        }
+
         private int DeleteDirect(string filePath)
         {
             using (log4net.ThreadContext.Stacks["StatusAgent"].Push("DeleteDirect"))
@@ -792,7 +820,7 @@ namespace Pithos.Core.Agents
             }
         }
 
-        public void UpdateFileChecksum(string path, string checksum)
+        public void UpdateFileChecksum(string path, string shortHash, string checksum)
         {
             if (String.IsNullOrWhiteSpace(path))
                 throw new ArgumentNullException("path");
@@ -800,7 +828,7 @@ namespace Pithos.Core.Agents
                 throw new ArgumentException("The path must be rooted", "path");            
             Contract.EndContractBlock();
 
-            _persistenceAgent.Post(() => FileState.UpdateChecksum(path, checksum));
+            _persistenceAgent.Post(() => FileState.UpdateChecksum(path, shortHash,checksum));
         }
 
     }
index 5fe0cda..7b796ec 100644 (file)
@@ -139,6 +139,13 @@ namespace Pithos.Core.Agents
                                 NetworkAgent.Post(new CloudDeleteAction(accountInfo, info, fileState));
                                 break;
                             case FileStatus.Renamed:
+                                if (state.OldPath == null)
+                                {
+                                    //We reach this point only if the app closed before propagating a rename to the server
+                                    Log.WarnFormat("Unfinished rename [{0}]",state.Path);
+                                    StatusKeeper.SetFileState(state.Path,FileStatus.Conflict,FileOverlayStatus.Conflict);
+                                    break;
+                                }
                                 FileSystemInfo oldInfo = Directory.Exists(state.OldPath)
                                                              ? (FileSystemInfo) new DirectoryInfo(state.OldPath)
                                                              : new FileInfo(state.OldPath);
index 5fbacbd..08e71f8 100644 (file)
@@ -81,8 +81,46 @@ namespace Pithos.Core
         [Property]
         public FileStatus FileStatus { get; set; }
 
+        private string _checksum;
+
+        /// <summary>
+        /// The tophash value of the file, calculated by using a Merkle hash with the SHA256 algorithm
+        /// </summary>
+        /// <remarks>
+        /// The SHA256 algorithm is substantially more expenive than other algorithms.
+        /// Recalculating the Checksum should be avoided whenever possible.
+        /// </remarks>
         [Property]
-        public string Checksum { get; set; }
+        public string Checksum
+        {
+            get
+            {
+                return _checksum;
+            }
+            set
+            {
+                _checksum = value;
+            }
+        }
+
+        private string _shortHash;
+
+        /// <summary>
+        /// An easy to calcualte hash over the entire file, used to detect file changes
+        /// </summary>
+        /// <remarks>The algorithm used to calculate this hash should be cheap</remarks>
+        [Property(NotNull=true,Default="")]
+        public string ShortHash
+        {
+            get
+            {
+                return _shortHash;
+            }
+            set
+            {
+                _shortHash = value;
+            }
+        }
 
 
         [Property]
@@ -202,7 +240,38 @@ namespace Pithos.Core
 
         }
 
-        public static void StoreOverlayStatus(string absolutePath, FileOverlayStatus newStatus)
+        /*public static void StoreOverlayStatus(string absolutePath, FileOverlayStatus newStatus)
+        {
+            if (string.IsNullOrWhiteSpace(absolutePath))
+                throw new ArgumentNullException("absolutePath");
+            Contract.EndContractBlock();
+
+            ExecuteWithRetry((session, instance) =>
+                        {
+                            const string hqlUpdate =
+                                "update FileState set OverlayStatus= :status where FilePath = :path ";
+                            var updatedEntities = session.CreateQuery(hqlUpdate)                                
+                                .SetString("path", absolutePath)
+                                .SetEnum("status", newStatus)                                
+                                .ExecuteUpdate();
+                            if (updatedEntities == 0)
+                            {
+                                var newState = new FileState
+                                                   {
+                                                       FilePath = absolutePath,
+                                                       Id = Guid.NewGuid(),
+                                                       OverlayStatus = newStatus,
+                                                       ShortHash = String.Empty,
+                                                       IsFolder=Directory.Exists(absolutePath)
+                                                   };
+                                newState.CreateAndFlush();
+                            }
+                            return null;
+                        }, null);
+
+        }
+*/
+        public static void StoreOverlayStatus(string absolutePath, FileOverlayStatus newStatus,string shortHash)
         {
             if (string.IsNullOrWhiteSpace(absolutePath))
                 throw new ArgumentNullException("absolutePath");
@@ -223,6 +292,7 @@ namespace Pithos.Core
                                                        FilePath = absolutePath,
                                                        Id = Guid.NewGuid(),
                                                        OverlayStatus = newStatus,
+                                                       ShortHash = shortHash??String.Empty,
                                                        IsFolder=Directory.Exists(absolutePath)
                                                    };
                                 newState.CreateAndFlush();
@@ -309,7 +379,7 @@ namespace Pithos.Core
             }, null);
         }*/
 
-        public static void UpdateChecksum(string absolutePath, string checksum)
+        public static void UpdateChecksum(string absolutePath, string shortHash, string checksum)
         {
             if (string.IsNullOrWhiteSpace(absolutePath))
                 throw new ArgumentNullException("absolutePath");
@@ -317,10 +387,11 @@ namespace Pithos.Core
 
             ExecuteWithRetry((session, instance) =>
                         {
-                            const string hqlUpdate = "update FileState set Checksum= :checksum where FilePath = :path ";
+                            const string hqlUpdate = "update FileState set Checksum= :checksum,ShortHash=:shortHash where FilePath = :path ";
                             var updatedEntities = session.CreateQuery(hqlUpdate)
                                 .SetString("path", absolutePath)
                                 .SetString("checksum", checksum)
+                                .SetString("shortHash", shortHash)
                                 .ExecuteUpdate();
                             return updatedEntities;
                         }, null);
@@ -357,49 +428,34 @@ namespace Pithos.Core
                             }, null);
         }
 
-        public static Task<FileState> CreateForAsync(string filePath, int blockSize, string algorithm)
+        public static FileState CreateFor(FileSystemInfo info)
         {
-            if (blockSize <= 0)
-                throw new ArgumentOutOfRangeException("blockSize");
-            if (String.IsNullOrWhiteSpace(algorithm))
-                throw new ArgumentNullException("algorithm");
+            Contract.Requires(info!=null);
             Contract.EndContractBlock();
+            
+            if (info is DirectoryInfo)
+                return new FileState
+                {
+                    FilePath = info.FullName,
+                    OverlayStatus = FileOverlayStatus.Unversioned,
+                    FileStatus = FileStatus.Created,
+                    ShortHash=String.Empty,
+                    Id = Guid.NewGuid()
+                };
 
 
+            var shortHash = ((FileInfo)info).ComputeShortHash();
             var fileState = new FileState
                                 {
-                                    FilePath = filePath,
+                                    FilePath = info.FullName,
                                     OverlayStatus = FileOverlayStatus.Unversioned,
-                                    FileStatus = FileStatus.Created,
+                                    FileStatus = FileStatus.Created,               
+                                    ShortHash=shortHash,
                                     Id = Guid.NewGuid()
                                 };
-
-
-            return fileState.UpdateHashesAsync(blockSize, algorithm);
+            return fileState;
         }
 
-        public async Task<FileState> UpdateHashesAsync(int blockSize, string algorithm)
-        {
-            if (blockSize <= 0)
-                throw new ArgumentOutOfRangeException("blockSize");
-            if (String.IsNullOrWhiteSpace(algorithm))
-                throw new ArgumentNullException("algorithm");
-            Contract.EndContractBlock();
-
-            //Skip updating the hash for folders
-            if (Directory.Exists(FilePath))
-                return this;
-
-            var hash = await TaskEx.Run(() =>
-                                            {
-                                                var info = new FileInfo(FilePath);
-                                                return info.CalculateHash(blockSize, algorithm);
-                                            });
-
-            Checksum = hash;
-
-            return this;
-        }
 
         private static void ExecuteWithRetry(NHibernateDelegate call, object state)
         {
index fa707dc..fb4cdab 100644 (file)
@@ -45,6 +45,7 @@ using System.Diagnostics.Contracts;
 using System.IO;
 using System.Linq;
 using System.Threading;
+using System.Threading.Tasks;
 using Pithos.Interfaces;
 
 namespace Pithos.Core
@@ -52,12 +53,11 @@ namespace Pithos.Core
     [ContractClass(typeof(IStatusKeeperContract))]
     public interface IStatusKeeper
     {
-        void SetFileOverlayStatus(string path,FileOverlayStatus status);
-        void UpdateFileChecksum(string path, string checksum);
+        Task SetFileOverlayStatus(string path, FileOverlayStatus status, string shortHash = null);
+        void UpdateFileChecksum(string path, string shortHash, string checksum);
         void SetFileStatus(string path, FileStatus status);
         FileStatus GetFileStatus(string path);
         void ClearFileStatus(string path);
-        void SetPithosStatus(PithosStatus status);
         FileOverlayStatus GetFileOverlayStatus(string path);
         void ProcessExistingFiles(IEnumerable<FileInfo> paths);
         void Stop();
@@ -74,18 +74,20 @@ namespace Pithos.Core
         FileState GetStateByFilePath(string path);
         void ClearFolderStatus(string path);
         IEnumerable<FileState> GetChildren(FileState fileState);
+        void EnsureFileState(string path);
     }
 
     [ContractClassFor(typeof(IStatusKeeper))]
     public abstract class IStatusKeeperContract : IStatusKeeper
     {
-        public void SetFileOverlayStatus(string path, FileOverlayStatus status)
+        public Task SetFileOverlayStatus(string path, FileOverlayStatus status, string shortHash = null)
         {
             Contract.Requires(!String.IsNullOrWhiteSpace(path));
             Contract.Requires(Path.IsPathRooted(path));
+            return default(Task);
         }
 
-        public void UpdateFileChecksum(string path, string checksum)
+        public void UpdateFileChecksum(string path, string shortHash, string checksum)
         {
             Contract.Requires(!String.IsNullOrWhiteSpace(path));
             Contract.Requires(checksum!=null);
@@ -228,5 +230,10 @@ namespace Pithos.Core
             Contract.Ensures(Contract.Result<IEnumerable<FileState>>()!=null);
             return default(IEnumerable<FileState>);
         }
+
+        public void EnsureFileState(string path)
+        {
+            
+        }
     }
 }
index f2fb632..201d37d 100644 (file)
@@ -55,6 +55,8 @@ namespace Pithos.Core
         void NotifyConflicts(IEnumerable<FileSystemInfo> conflictFiles, string message);\r
         void NotifyForFiles(IEnumerable<FileSystemInfo> files, string message,TraceLevel level);\r
         void Notify(Notification notification);\r
+        void SetPithosStatus(PithosStatus status);\r
+        void SetPithosStatus(PithosStatus localSyncing, string format);\r
     }\r
 \r
     public class Notification\r
@@ -71,6 +73,15 @@ namespace Pithos.Core
         }\r
     }\r
 \r
+    public class StatusNotification:Notification\r
+    {\r
+        public StatusNotification(string title)\r
+        {\r
+            Level = TraceLevel.Info;\r
+            Title = title;\r
+        }\r
+    }\r
+\r
     public class ProgressNotification:Notification\r
     {\r
         public string FileName { get; set; }\r
index f156832..0fd61f7 100644 (file)
     <CodeContractsArithmeticObligations>True</CodeContractsArithmeticObligations>
     <CodeContractsEnumObligations>True</CodeContractsEnumObligations>
     <CodeContractsRedundantAssumptions>False</CodeContractsRedundantAssumptions>
-    <CodeContractsRunInBackground>True</CodeContractsRunInBackground>
+    <CodeContractsRunInBackground>False</CodeContractsRunInBackground>
     <CodeContractsShowSquigglies>True</CodeContractsShowSquigglies>
     <CodeContractsUseBaseLine>False</CodeContractsUseBaseLine>
     <CodeContractsEmitXMLDocs>True</CodeContractsEmitXMLDocs>
   <ItemGroup>
     <Compile Include="Agents\Agent.cs" />
     <Compile Include="Agents\AgentLocator.cs" />
+    <Compile Include="Agents\AsyncCollection.cs" />
     <Compile Include="Agents\AsyncManualResetEvent.cs" />
+    <Compile Include="Agents\AsyncSemaphore.cs" />
     <Compile Include="Agents\BlockUpdater.cs" />
     <Compile Include="Agents\CloudTransferAction.cs" />
     <Compile Include="Agents\CollectionExtensions.cs" />
index b366804..b0a1abf 100644 (file)
@@ -144,16 +144,6 @@ namespace Pithos.Core
             set
             {
                 FileAgent.Pause = value;
-                if (value)
-                {
-                    StatusKeeper.SetPithosStatus(PithosStatus.SyncPaused);
-                    StatusNotification.NotifyChange("Paused");
-                }
-                else
-                {
-                    StatusKeeper.SetPithosStatus(PithosStatus.InSynch);
-                    StatusNotification.NotifyChange("Synchronizing");
-                }
             }
         }
 
@@ -268,12 +258,15 @@ namespace Pithos.Core
 
         private void IndexLocalFiles()
         {
-            StatusNotification.NotifyChange("Indexing Local Files");
             using (ThreadContext.Stacks["Operation"].Push("Indexing local files"))
             {
-                Log.Info("START");
+                
                 try
                 {
+                    //StatusNotification.NotifyChange("Indexing Local Files");
+                    Log.Info("Start local indexing");
+                    StatusNotification.SetPithosStatus(PithosStatus.LocalSyncing,"Indexing Local Files");                    
+
                     var cachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
                     var directory = new DirectoryInfo(RootPath);
                     var files =
@@ -292,6 +285,7 @@ namespace Pithos.Core
                 {
                     Log.Info("[END]");
                 }
+                StatusNotification.SetPithosStatus(PithosStatus.LocalComplete,"Indexing Completed");
             }
         }
 
@@ -360,11 +354,13 @@ namespace Pithos.Core
             }
         }        
 
+
         private void StartNetworkAgent()
         {
-
             NetworkAgent.StatusNotification = StatusNotification;
-                        
+
+            //TODO: The Network and Poll agents are not account specific
+            //They should be moved outside PithosMonitor
             NetworkAgent.Start();
 
             PollAgent.AddAccount(_accountInfo);
@@ -409,9 +405,13 @@ namespace Pithos.Core
 
         private void StartWatcherAgent()
         {
+            if (Log.IsDebugEnabled)
+                Log.DebugFormat("Start Folder Monitoring [{0}]",RootPath);
+
             AgentLocator<FileAgent>.Register(FileAgent,RootPath);
 
             FileAgent.StatusKeeper = StatusKeeper;
+            FileAgent.StatusNotification = StatusNotification;
             FileAgent.Workflow = Workflow;
             FileAgent.CachePath = Path.Combine(RootPath, FolderConstants.CacheFolder);
             FileAgent.Start(_accountInfo, RootPath);
index 7698ee5..68b799e 100644 (file)
@@ -41,6 +41,7 @@
 #endregion
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Contracts;
 using System.IO;
 using System.Linq;
 using System.Text;
@@ -64,8 +65,9 @@ namespace Pithos.Core
         
         public bool Skip { get; set; }
 
-        public string Hash { get; set; }
+        public string Hash { get; set; }        
         public string LastUpdateHash { get; set; }
+        public string ShortHash { get; set; }
 
 /*
         public WorkflowState(AccountInfo accountInfo)
@@ -81,10 +83,17 @@ namespace Pithos.Core
 
         public WorkflowState(AccountInfo accountInfo, FileState state)
         {
+            if (accountInfo==null)
+                throw new ArgumentNullException("accountInfo");
+            if (state==null)
+                throw new ArgumentNullException("state");
+            Contract.EndContractBlock();
+
             AccountInfo = accountInfo;
             Path = state.FilePath.ToLower();
             FileName = System.IO.Path.GetFileName(state.FilePath).ToLower();
             Hash = state.Checksum;
+            ShortHash = state.ShortHash;
             Status = state.OverlayStatus == FileOverlayStatus.Unversioned
                          ? FileStatus.Created
                          : state.FileStatus;
index e701c5d..4a9115e 100644 (file)
@@ -54,8 +54,6 @@ namespace Pithos.Interfaces
     {
         FileOverlayStatus GetFileOverlayStatus(string path);
 
-        PithosStatus GetPithosStatus();        
-
     }
 
     [ContractClassFor(typeof(IStatusChecker))]
@@ -87,7 +85,10 @@ namespace Pithos.Interfaces
     public enum PithosStatus
     {
         InSynch,
-        Syncing,
+        PollSyncing,
+        PollComplete,
+        LocalSyncing,
+        LocalComplete,
         SyncPaused,
         HasConflicts,
         Disconnected
index f96b6f5..a450690 100644 (file)
     <CodeContractsArithmeticObligations>True</CodeContractsArithmeticObligations>
     <CodeContractsEnumObligations>True</CodeContractsEnumObligations>
     <CodeContractsRedundantAssumptions>False</CodeContractsRedundantAssumptions>
-    <CodeContractsRunInBackground>True</CodeContractsRunInBackground>
+    <CodeContractsRunInBackground>False</CodeContractsRunInBackground>
     <CodeContractsShowSquigglies>True</CodeContractsShowSquigglies>
     <CodeContractsUseBaseLine>False</CodeContractsUseBaseLine>
     <CodeContractsEmitXMLDocs>True</CodeContractsEmitXMLDocs>
index 4ad29ef..513e2db 100644 (file)
     <CodeContractsArithmeticObligations>True</CodeContractsArithmeticObligations>
     <CodeContractsEnumObligations>True</CodeContractsEnumObligations>
     <CodeContractsRedundantAssumptions>False</CodeContractsRedundantAssumptions>
-    <CodeContractsRunInBackground>True</CodeContractsRunInBackground>
+    <CodeContractsRunInBackground>False</CodeContractsRunInBackground>
     <CodeContractsShowSquigglies>True</CodeContractsShowSquigglies>
     <CodeContractsUseBaseLine>False</CodeContractsUseBaseLine>
     <CodeContractsEmitXMLDocs>True</CodeContractsEmitXMLDocs>
index e696bd3..634fea5 100644 (file)
@@ -115,18 +115,21 @@ namespace Pithos.Network
             return shb.ToString().ToLower();
         }
 
-        public static TreeHash CalculateTreeHash(FileInfo fileInfo, int blockSize, string algorithm)
+        public static TreeHash CalculateTreeHash(FileSystemInfo fileInfo, int blockSize, string algorithm)
         {
-            if (fileInfo==null)
+            if (fileInfo == null)
                 throw new ArgumentNullException("fileInfo");
             if (String.IsNullOrWhiteSpace(fileInfo.FullName))
-                throw new ArgumentException("fileInfo.FullName is empty","fileInfo");
+                throw new ArgumentException("fileInfo.FullName is empty", "fileInfo");
             if (blockSize <= 0)
                 throw new ArgumentOutOfRangeException("blockSize", "blockSize must be a value greater than zero ");
             if (String.IsNullOrWhiteSpace(algorithm))
                 throw new ArgumentNullException("algorithm");
             Contract.EndContractBlock();
 
+            if (fileInfo is DirectoryInfo || !fileInfo.Exists)
+                return TreeHash.Empty;
+
             return CalculateTreeHash(fileInfo.FullName, blockSize, algorithm);
         }
 
@@ -177,6 +180,9 @@ namespace Pithos.Network
                 throw new ArgumentNullException("algorithm");
             Contract.EndContractBlock();
 
+            if (Log.IsDebugEnabled)
+                Log.DebugFormat("Calc Signature [{0}]",filePath);
+
             //DON'T calculate hashes for folders
             if (Directory.Exists(filePath))
                 return new TreeHash(algorithm);
index 88c5d04..b0bf7bf 100644 (file)
@@ -87,6 +87,7 @@ Global
                {240B432F-1030-4623-BCC3-FF351D6C1B63}.Premium Debug|x64.ActiveCfg = Premium Debug|x64
                {240B432F-1030-4623-BCC3-FF351D6C1B63}.Premium Debug|x64.Build.0 = Premium Debug|x64
                {240B432F-1030-4623-BCC3-FF351D6C1B63}.Premium Debug|x86.ActiveCfg = Premium Debug|Any CPU
+               {240B432F-1030-4623-BCC3-FF351D6C1B63}.Premium Debug|x86.Build.0 = Premium Debug|Any CPU
                {240B432F-1030-4623-BCC3-FF351D6C1B63}.Release|Any CPU.ActiveCfg = Release|Any CPU
                {240B432F-1030-4623-BCC3-FF351D6C1B63}.Release|Any CPU.Build.0 = Release|Any CPU
                {240B432F-1030-4623-BCC3-FF351D6C1B63}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
@@ -118,6 +119,7 @@ Global
                {2CFE2DF1-20AE-47E2-B1BB-36B974600BE1}.Premium Debug|Mixed Platforms.Build.0 = Premium Debug|Any CPU
                {2CFE2DF1-20AE-47E2-B1BB-36B974600BE1}.Premium Debug|x64.ActiveCfg = Premium Debug|x64
                {2CFE2DF1-20AE-47E2-B1BB-36B974600BE1}.Premium Debug|x86.ActiveCfg = Premium Debug|Any CPU
+               {2CFE2DF1-20AE-47E2-B1BB-36B974600BE1}.Premium Debug|x86.Build.0 = Premium Debug|Any CPU
                {2CFE2DF1-20AE-47E2-B1BB-36B974600BE1}.Release|Any CPU.ActiveCfg = Release|Any CPU
                {2CFE2DF1-20AE-47E2-B1BB-36B974600BE1}.Release|Any CPU.Build.0 = Release|Any CPU
                {2CFE2DF1-20AE-47E2-B1BB-36B974600BE1}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
@@ -151,6 +153,7 @@ Global
                {7EEFF32F-CCF8-436A-9E0B-F40434C09AF4}.Premium Debug|x64.ActiveCfg = Premium Debug|x64
                {7EEFF32F-CCF8-436A-9E0B-F40434C09AF4}.Premium Debug|x64.Build.0 = Premium Debug|x64
                {7EEFF32F-CCF8-436A-9E0B-F40434C09AF4}.Premium Debug|x86.ActiveCfg = Premium Debug|Any CPU
+               {7EEFF32F-CCF8-436A-9E0B-F40434C09AF4}.Premium Debug|x86.Build.0 = Premium Debug|Any CPU
                {7EEFF32F-CCF8-436A-9E0B-F40434C09AF4}.Release|Any CPU.ActiveCfg = Release|Any CPU
                {7EEFF32F-CCF8-436A-9E0B-F40434C09AF4}.Release|Any CPU.Build.0 = Release|Any CPU
                {7EEFF32F-CCF8-436A-9E0B-F40434C09AF4}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
@@ -187,6 +190,7 @@ Global
                {822F885B-83E8-4A9A-B02E-0FEAE444D960}.Premium Debug|x64.ActiveCfg = Premium Debug|x64
                {822F885B-83E8-4A9A-B02E-0FEAE444D960}.Premium Debug|x64.Build.0 = Premium Debug|x64
                {822F885B-83E8-4A9A-B02E-0FEAE444D960}.Premium Debug|x86.ActiveCfg = Premium Debug|Any CPU
+               {822F885B-83E8-4A9A-B02E-0FEAE444D960}.Premium Debug|x86.Build.0 = Premium Debug|Any CPU
                {822F885B-83E8-4A9A-B02E-0FEAE444D960}.Release|Any CPU.ActiveCfg = Release|Any CPU
                {822F885B-83E8-4A9A-B02E-0FEAE444D960}.Release|Any CPU.Build.0 = Release|Any CPU
                {822F885B-83E8-4A9A-B02E-0FEAE444D960}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
@@ -223,6 +227,7 @@ Global
                {C45218F8-09E7-4F57-85BC-5D8D2AC736A3}.Premium Debug|x64.ActiveCfg = Premium Debug|x64
                {C45218F8-09E7-4F57-85BC-5D8D2AC736A3}.Premium Debug|x64.Build.0 = Premium Debug|x64
                {C45218F8-09E7-4F57-85BC-5D8D2AC736A3}.Premium Debug|x86.ActiveCfg = Premium Debug|Any CPU
+               {C45218F8-09E7-4F57-85BC-5D8D2AC736A3}.Premium Debug|x86.Build.0 = Premium Debug|Any CPU
                {C45218F8-09E7-4F57-85BC-5D8D2AC736A3}.Release|Any CPU.ActiveCfg = Release|Any CPU
                {C45218F8-09E7-4F57-85BC-5D8D2AC736A3}.Release|Any CPU.Build.0 = Release|Any CPU
                {C45218F8-09E7-4F57-85BC-5D8D2AC736A3}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
@@ -259,6 +264,7 @@ Global
                {142AF135-DF30-4563-B0AC-B604235AE874}.Premium Debug|x64.ActiveCfg = Premium Debug|x64
                {142AF135-DF30-4563-B0AC-B604235AE874}.Premium Debug|x64.Build.0 = Premium Debug|x64
                {142AF135-DF30-4563-B0AC-B604235AE874}.Premium Debug|x86.ActiveCfg = Premium Debug|Any CPU
+               {142AF135-DF30-4563-B0AC-B604235AE874}.Premium Debug|x86.Build.0 = Premium Debug|Any CPU
                {142AF135-DF30-4563-B0AC-B604235AE874}.Release|Any CPU.ActiveCfg = Release|Any CPU
                {142AF135-DF30-4563-B0AC-B604235AE874}.Release|Any CPU.Build.0 = Release|Any CPU
                {142AF135-DF30-4563-B0AC-B604235AE874}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
@@ -295,6 +301,7 @@ Global
                {E027200B-C26A-4877-BFD9-1A18CF5DF2F4}.Premium Debug|x64.ActiveCfg = Premium Debug|x64
                {E027200B-C26A-4877-BFD9-1A18CF5DF2F4}.Premium Debug|x64.Build.0 = Premium Debug|x64
                {E027200B-C26A-4877-BFD9-1A18CF5DF2F4}.Premium Debug|x86.ActiveCfg = Premium Debug|Any CPU
+               {E027200B-C26A-4877-BFD9-1A18CF5DF2F4}.Premium Debug|x86.Build.0 = Premium Debug|Any CPU
                {E027200B-C26A-4877-BFD9-1A18CF5DF2F4}.Release|Any CPU.ActiveCfg = Release|Any CPU
                {E027200B-C26A-4877-BFD9-1A18CF5DF2F4}.Release|Any CPU.Build.0 = Release|Any CPU
                {E027200B-C26A-4877-BFD9-1A18CF5DF2F4}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
@@ -331,6 +338,7 @@ Global
                {C8E2BC8B-C7F1-4222-855C-4B04A57FFDFD}.Premium Debug|x64.ActiveCfg = Premium Debug|x64
                {C8E2BC8B-C7F1-4222-855C-4B04A57FFDFD}.Premium Debug|x64.Build.0 = Premium Debug|x64
                {C8E2BC8B-C7F1-4222-855C-4B04A57FFDFD}.Premium Debug|x86.ActiveCfg = Premium Debug|Any CPU
+               {C8E2BC8B-C7F1-4222-855C-4B04A57FFDFD}.Premium Debug|x86.Build.0 = Premium Debug|Any CPU
                {C8E2BC8B-C7F1-4222-855C-4B04A57FFDFD}.Release|Any CPU.ActiveCfg = Release|Any CPU
                {C8E2BC8B-C7F1-4222-855C-4B04A57FFDFD}.Release|Any CPU.Build.0 = Release|Any CPU
                {C8E2BC8B-C7F1-4222-855C-4B04A57FFDFD}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
@@ -367,6 +375,7 @@ Global
                {F9AF3E97-BCB7-46B7-8014-7FC858AEE9BA}.Premium Debug|x64.ActiveCfg = Premium Debug|x64
                {F9AF3E97-BCB7-46B7-8014-7FC858AEE9BA}.Premium Debug|x64.Build.0 = Premium Debug|x64
                {F9AF3E97-BCB7-46B7-8014-7FC858AEE9BA}.Premium Debug|x86.ActiveCfg = Premium Debug|Any CPU
+               {F9AF3E97-BCB7-46B7-8014-7FC858AEE9BA}.Premium Debug|x86.Build.0 = Premium Debug|Any CPU
                {F9AF3E97-BCB7-46B7-8014-7FC858AEE9BA}.Release|Any CPU.ActiveCfg = Release|Any CPU
                {F9AF3E97-BCB7-46B7-8014-7FC858AEE9BA}.Release|Any CPU.Build.0 = Release|Any CPU
                {F9AF3E97-BCB7-46B7-8014-7FC858AEE9BA}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
@@ -403,6 +412,7 @@ Global
                {A9AE40FF-1A21-414A-9FE7-3BE13644CC6D}.Premium Debug|x64.ActiveCfg = Premium Debug|x64
                {A9AE40FF-1A21-414A-9FE7-3BE13644CC6D}.Premium Debug|x64.Build.0 = Premium Debug|x64
                {A9AE40FF-1A21-414A-9FE7-3BE13644CC6D}.Premium Debug|x86.ActiveCfg = Premium Debug|Any CPU
+               {A9AE40FF-1A21-414A-9FE7-3BE13644CC6D}.Premium Debug|x86.Build.0 = Premium Debug|Any CPU
                {A9AE40FF-1A21-414A-9FE7-3BE13644CC6D}.Release|Any CPU.ActiveCfg = Release|Any CPU
                {A9AE40FF-1A21-414A-9FE7-3BE13644CC6D}.Release|Any CPU.Build.0 = Release|Any CPU
                {A9AE40FF-1A21-414A-9FE7-3BE13644CC6D}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
@@ -477,6 +487,7 @@ Global
                {7AC63864-7638-41C4-969C-D3197EF2BED9}.Premium Debug|x64.ActiveCfg = Premium Debug|x64
                {7AC63864-7638-41C4-969C-D3197EF2BED9}.Premium Debug|x64.Build.0 = Premium Debug|x64
                {7AC63864-7638-41C4-969C-D3197EF2BED9}.Premium Debug|x86.ActiveCfg = Premium Debug|Any CPU
+               {7AC63864-7638-41C4-969C-D3197EF2BED9}.Premium Debug|x86.Build.0 = Premium Debug|Any CPU
                {7AC63864-7638-41C4-969C-D3197EF2BED9}.Release|Any CPU.ActiveCfg = Release|Any CPU
                {7AC63864-7638-41C4-969C-D3197EF2BED9}.Release|Any CPU.Build.0 = Release|Any CPU
                {7AC63864-7638-41C4-969C-D3197EF2BED9}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
@@ -508,7 +519,6 @@ Global
                {C6251981-3C49-404B-BB5B-9732887388D2}.Premium Debug|x64.ActiveCfg = Premium Debug
                {C6251981-3C49-404B-BB5B-9732887388D2}.Premium Debug|x64.Build.0 = Premium Debug
                {C6251981-3C49-404B-BB5B-9732887388D2}.Premium Debug|x86.ActiveCfg = Premium Debug
-               {C6251981-3C49-404B-BB5B-9732887388D2}.Premium Debug|x86.Build.0 = Premium Debug
                {C6251981-3C49-404B-BB5B-9732887388D2}.Release|Any CPU.ActiveCfg = Release
                {C6251981-3C49-404B-BB5B-9732887388D2}.Release|Mixed Platforms.ActiveCfg = Release
                {C6251981-3C49-404B-BB5B-9732887388D2}.Release|x64.ActiveCfg = Release