public void SynchNow(IEnumerable<string> paths=null)\r
{\r
_batchQueue.Enqueue(paths);\r
- _syncEvent.Set(); \r
+ _syncEvent.SetAsync(); \r
+\r
+ //_pollAction.Post(new PollRequest {Batch = paths});\r
+ }\r
+\r
+ /// <summary>\r
+ /// Start a manual synchronization\r
+ /// </summary>\r
+ public Task SynchNowAsync(IEnumerable<string> paths=null)\r
+ {\r
+ _batchQueue.Enqueue(paths);\r
+ return _syncEvent.SetAsync(); \r
\r
//_pollAction.Post(new PollRequest {Batch = paths});\r
}\r
TaskEx.Run(() => _moves.AddOrUpdate(args.OldFullPath, args,(s,e)=>e)); \r
}\r
\r
+\r
+ private bool _hasConnection;\r
+\r
/// <summary>\r
/// Remote files are polled periodically. Any changes are processed\r
/// </summary>\r
/// <param name="since"></param>\r
/// <returns></returns>\r
- public void PollRemoteFiles(DateTime? since = null)\r
+ public async Task PollRemoteFiles(DateTimeOffset? since = null)\r
{\r
if (Log.IsDebugEnabled)\r
Log.DebugFormat("Polling changes after [{0}]",since);\r
\r
//GC.Collect();\r
\r
+ \r
using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts"))\r
{\r
//If this poll fails, we will retry with the same since value\r
- var nextSince = since;\r
+ DateTimeOffset? nextSince = since;\r
try\r
{\r
_unPauseEvent.Wait();\r
UpdateStatus(PithosStatus.PollSyncing);\r
\r
- var accountBatches=new Dictionary<Uri, IEnumerable<string>>();\r
- IEnumerable<string> batch = null;\r
- if (_batchQueue.TryDequeue(out batch) && batch != null)\r
- foreach (var account in _accounts.Values)\r
+ if (!NetworkAgent.IsConnectedToInternet)\r
+ {\r
+ if (_hasConnection)\r
{\r
- var accountBatch = batch.Where(path => path.IsAtOrBelow(account.AccountPath));\r
- accountBatches[account.AccountKey] = accountBatch;\r
+ StatusNotification.Notify(new Notification\r
+ {\r
+ Level = TraceLevel.Error,\r
+ Title = "Internet Connection problem",\r
+ Message ="Internet connectivity was lost. Synchronization will continue when connectivity is restored"\r
+ });\r
}\r
- \r
- var moves=Interlocked.Exchange(ref _moves, new ConcurrentDictionary<string, MovedEventArgs>());\r
-\r
- var tasks = new List<Task<DateTime?>>();\r
- foreach(var accountInfo in _accounts.Values)\r
- {\r
- IEnumerable<string> accountBatch ;\r
- accountBatches.TryGetValue(accountInfo.AccountKey,out accountBatch);\r
- var t=ProcessAccountFiles (accountInfo, accountBatch, moves,since);\r
- tasks.Add(t);\r
+ _hasConnection = false;\r
}\r
+ else\r
+ {\r
+ if (!_hasConnection)\r
+ {\r
+ StatusNotification.Notify(new Notification\r
+ {\r
+ Level = TraceLevel.Info,\r
+ Title = "Internet Connection",\r
+ Message = "Internet connectivity restored."\r
+ });\r
+ }\r
+ _hasConnection = true;\r
+\r
+ var accountBatches = new Dictionary<Uri, IEnumerable<string>>();\r
+ IEnumerable<string> batch = null;\r
+ if (_batchQueue.TryDequeue(out batch) && batch != null)\r
+ foreach (var account in _accounts.Values)\r
+ {\r
+ var accountBatch = batch.Where(path => path.IsAtOrBelow(account.AccountPath));\r
+ accountBatches[account.AccountKey] = accountBatch;\r
+ }\r
+\r
+ var moves = Interlocked.Exchange(ref _moves, new ConcurrentDictionary<string, MovedEventArgs>());\r
+\r
+ var tasks = new List<Task<DateTimeOffset?>>();\r
+ foreach (var accountInfo in _accounts.Values)\r
+ {\r
+ IEnumerable<string> accountBatch;\r
+ accountBatches.TryGetValue(accountInfo.AccountKey, out accountBatch);\r
+ var t = ProcessAccountFiles(accountInfo, accountBatch, moves, since);\r
+ tasks.Add(t);\r
+ }\r
\r
- var nextTimes=TaskEx.WhenAll(tasks.ToList()).Result;\r
+ var taskList = tasks.ToList();\r
+ var nextTimes = await TaskEx.WhenAll(taskList).ConfigureAwait(false);\r
\r
- _firstPoll = false;\r
- //Reschedule the poll with the current timestamp as a "since" value\r
+ _firstPoll = false;\r
+ //Reschedule the poll with the current timestamp as a "since" value\r
\r
- if (nextTimes.Length>0)\r
- nextSince = nextTimes.Min();\r
- if (Log.IsDebugEnabled)\r
- Log.DebugFormat("Next Poll for changes since [{0}]",nextSince);\r
+ if (nextTimes.Length > 0)\r
+ nextSince = nextTimes.Min();\r
+ if (Log.IsDebugEnabled)\r
+ Log.DebugFormat("Next Poll for changes since [{0}]", nextSince);\r
+ }\r
}\r
catch (Exception ex)\r
{\r
try\r
{\r
//Wait for the polling interval to pass or the Sync event to be signalled\r
- nextSince = WaitForScheduledOrManualPoll(nextSince).Result;\r
+ nextSince = await WaitForScheduledOrManualPoll(nextSince).ConfigureAwait(false);\r
}\r
finally\r
{\r
//Ensure polling is scheduled even in case of error\r
+#pragma warning disable 4014\r
TaskEx.Run(()=>PollRemoteFiles(nextSince));\r
+#pragma warning restore 4014\r
//_pollAction.Post(new PollRequest {Since = nextSince});\r
}\r
}\r
/// </summary>\r
/// <param name="since"></param>\r
/// <returns></returns>\r
- private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since)\r
+ private async Task<DateTimeOffset?> WaitForScheduledOrManualPoll(DateTimeOffset? since)\r
{\r
var sync = _syncEvent.WaitAsync();\r
var delay = TimeSpan.FromSeconds(Settings.PollingInterval);\r
\r
\r
\r
- public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, IEnumerable<string> accountBatch, ConcurrentDictionary<string, MovedEventArgs> moves, DateTime? since = null)\r
+ public async Task<DateTimeOffset?> ProcessAccountFiles(AccountInfo accountInfo, IEnumerable<string> accountBatch, ConcurrentDictionary<string, MovedEventArgs> moves, DateTimeOffset? since = null)\r
{\r
if (accountInfo == null)\r
throw new ArgumentNullException("accountInfo");\r
var client = new CloudFilesClient(accountInfo);\r
\r
//We don't need to check the trash container\r
- var containers = client.ListContainers(accountInfo.UserName)\r
+ var allContainers=await client.ListContainers(accountInfo.UserName).ConfigureAwait(false);\r
+ var containers = allContainers\r
.Where(c=>c.Name.ToString()!="trash")\r
.ToList();\r
\r
//The nextSince time fallback time is the same as the current.\r
//If polling succeeds, the next Since time will be the smallest of the maximum modification times\r
//of the shared and account objects\r
- var nextSince = since;\r
+ DateTimeOffset? nextSince = since;\r
\r
try\r
{\r
//Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted\r
//than delete a file that was created while we were executing the poll \r
\r
+ var token = _currentOperationCancellation.Token;\r
+\r
//Get the list of server objects changed since the last check\r
//The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step\r
var listObjects = (from container in containers\r
select Task<IList<ObjectInfo>>.Factory.StartNew(_ =>\r
- client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList();\r
+ client.ListObjects(accountInfo.UserName, container.Name, since), container.Name,token)).ToList();\r
\r
- var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
- client.ListSharedObjects(_knownContainers,since), "shared");\r
+ var selectiveEnabled = Selectives.IsSelectiveEnabled(accountInfo.AccountKey);\r
+ var listShared = selectiveEnabled?\r
+ Task<IList<ObjectInfo>>.Factory.StartNew(_ => \r
+ client.ListSharedObjects(_knownContainers,since), "shared",token)\r
+ :Task.Factory.FromResult((IList<ObjectInfo>) new List<ObjectInfo>(),"shared");\r
+ \r
listObjects.Add(listShared);\r
var listTasks = await Task.Factory.WhenAll(listObjects.ToArray()).ConfigureAwait(false);\r
\r
using (ThreadContext.Stacks["SCHEDULE"].Push("Process Results"))\r
{\r
+\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
var dict = listTasks.ToDictionary(t => t.AsyncState);\r
\r
//Get all non-trash objects. Remember, the container name is stored in AsyncState\r
var remoteObjects = (from objectList in listTasks\r
- where (string)objectList.AsyncState.ToString() != "trash"\r
+ where objectList.AsyncState.ToString() != "trash"\r
from obj in objectList.Result\r
orderby obj.Bytes ascending \r
select obj).ToList();\r
StringComparison.InvariantCultureIgnoreCase)\r
select info).ToList();\r
\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
if (_firstPoll)\r
StatusKeeper.CleanupOrphanStates();\r
\r
var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
var currentRemotes = differencer.Current.ToList();\r
+\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+ \r
StatusKeeper.CleanupStaleStates(accountInfo, currentRemotes);\r
\r
//var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
//May have to wait if the FileAgent has asked for a Pause, due to local changes\r
await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
//Get the local files here \r
var agent = AgentLocator<FileAgent>.Get(accountInfo.AccountPath); \r
var files = LoadLocalFileTuples(accountInfo, accountBatch);\r
\r
\r
- var states = StatusKeeper.GetAllStates();\r
- \r
+ \r
+ //WARNING: GetFileSystemInfo may create state entries.\r
+ //TODO: Find a different way to create the tuples and block long filenames\r
var infos = (from remote in currentRemotes\r
let path = remote.RelativeUrlToFilePath(accountInfo.UserName)\r
let info=agent.GetFileSystemInfo(path)\r
+ where info != null\r
select Tuple.Create(info.FullName,remote))\r
.ToList();\r
\r
- var token = _currentOperationCancellation.Token;\r
+ var states = StatusKeeper.GetAllStates();\r
\r
- var tuples = MergeSources(infos, files, states,moves).ToList();\r
+ var tupleBuilder = new TupleBuilder(CancellationToken,StatusKeeper,StatusNotification,Settings);\r
+\r
+ var tuples = tupleBuilder.MergeSources(infos, files, states,moves).ToList();\r
\r
var processedPaths = new HashSet<string>();\r
//Process only the changes in the batch file, if one exists\r
{\r
await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
\r
+ //In case of cancellation, retry for the current date\r
+ if (token.IsCancellationRequested) return since;\r
+\r
//Set the Merkle Hash\r
//SetMerkleHash(accountInfo, tuple);\r
\r
{\r
// No server changes\r
//Has the file been renamed locally?\r
- if (!MoveForLocalMove(accountInfo,tuple))\r
+ if (!await MoveForLocalMove(accountInfo,tuple))\r
//Has the file been renamed on the server?\r
MoveForServerMove(accountInfo, tuple);\r
}\r
//Server file doesn't exist\r
//deleteObjectFromLocal()\r
using (\r
- StatusNotification.GetNotifier("Deleting local {0}", "Deleted local {0}",\r
+ StatusNotification.GetNotifier("Deleting local {0}", "Deleted local {0}",true,\r
localInfo.Name))\r
{\r
DeleteLocalFile(agent, localFilePath);\r
//downloadServerObject() // Result: L = S\r
//If the file has moved on the server, move it locally before downloading\r
using (\r
- StatusNotification.GetNotifier("Downloading {0}", "Downloaded {0}",\r
+ StatusNotification.GetNotifier("Downloading {0}", "Downloaded {0}",true,\r
localInfo.Name))\r
{\r
var targetPath = MoveForServerMove(accountInfo, tuple);\r
AddOwnFolderToSelectives(accountInfo, tuple, targetPath);\r
}\r
}\r
-\r
- /*\r
- StatusKeeper.SetFileState(targetPath, FileStatus.Unchanged,\r
- FileOverlayStatus.Normal, "");\r
- */\r
}\r
}\r
}\r
else\r
{\r
//uploadLocalObject() // Result: S = C, L = S \r
- var progress = new Progress<double>(d =>\r
- StatusNotification.Notify(new StatusNotification(String.Format("Merkle Hashing for Upload {0:p} of {1}", d, localInfo.Name))));\r
+ var progress = new Progress<HashProgress>(d =>\r
+ StatusNotification.Notify(new StatusNotification(String.Format("Merkle Hashing for Upload {0:p} of {1}", d.Percentage, localInfo.Name))));\r
\r
//Is it an unselected root folder\r
var isCreation = isUnselectedRootFolder ||//or a new folder under a selected parent?\r
\r
//Is this a result of a FILE move with no modifications? Then try to move it,\r
//to avoid an expensive hash\r
- if (!MoveForLocalMove(accountInfo, tuple))\r
+ if (!await MoveForLocalMove(accountInfo, tuple))\r
{\r
await UploadLocalFile(accountInfo, tuple, token, isCreation, localInfo,processedPaths, progress).ConfigureAwait(false);\r
}\r
}\r
\r
private async Task DownloadCloudFile(AccountInfo accountInfo, StateTuple tuple, CancellationToken token, string targetPath)\r
- {\r
- StatusKeeper.SetFileState(targetPath, FileStatus.Modified, FileOverlayStatus.Modified,\r
- "");\r
+ { \r
+ //Don't create a new state for non-existent files\r
+ if (File.Exists(targetPath) || Directory.Exists(targetPath))\r
+ StatusKeeper.SetFileState(targetPath, FileStatus.Modified, FileOverlayStatus.Modified,"");\r
\r
var finalHash=await\r
NetworkAgent.Downloader.DownloadCloudFile(accountInfo, tuple.ObjectInfo, targetPath,\r
}\r
\r
private async Task UploadLocalFile(AccountInfo accountInfo, StateTuple tuple, CancellationToken token,\r
- bool isUnselectedRootFolder, FileSystemInfo localInfo, HashSet<string> processedPaths, Progress<double> progress)\r
+ bool isUnselectedRootFolder, FileSystemInfo localInfo, HashSet<string> processedPaths, IProgress<HashProgress> progress)\r
{\r
var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
accountInfo.BlockSize, accountInfo.BlockHash,\r
"Poll", isUnselectedRootFolder, token, progress,tuple.Merkle); \r
- \r
- using (StatusNotification.GetNotifier("Uploading {0}", "Uploaded {0}",\r
+\r
+ using (StatusNotification.GetNotifier("Uploading {0}", "Uploaded {0}",true,\r
localInfo.Name))\r
{\r
await NetworkAgent.Uploader.UploadCloudFile(action, token).ConfigureAwait(false);\r
}\r
}\r
\r
- private bool MoveForLocalMove(AccountInfo accountInfo, StateTuple tuple)\r
+ private async Task<bool> MoveForLocalMove(AccountInfo accountInfo, StateTuple tuple)\r
{\r
- //Is the file a directory or previous path missing?\r
- if (tuple.FileInfo is DirectoryInfo)\r
- return false;\r
//Is the previous path missing?\r
if (String.IsNullOrWhiteSpace(tuple.OldFullPath))\r
return false;\r
\r
var client = new CloudFilesClient(accountInfo);\r
var objectInfo = CloudAction.CreateObjectInfoFor(accountInfo, tuple.FileInfo);\r
+ objectInfo.X_Object_Hash = tuple.Merkle.TopHash.ToHashString();\r
var containerPath = Path.Combine(accountInfo.AccountPath, objectInfo.Container.ToUnescapedString());\r
//TODO: SImplify these multiple conversions from and to Uris\r
var oldName = tuple.OldFullPath.AsRelativeTo(containerPath);\r
//Then execute a move instead of an upload\r
- using (StatusNotification.GetNotifier("Moving {0}", "Moved {0}", tuple.FileInfo.Name))\r
+ using (StatusNotification.GetNotifier("Moving {0}", "Moved {0}", true,tuple.FileInfo.Name))\r
{\r
- client.MoveObject(objectInfo.Account, objectInfo.Container, oldName.ToEscapedUri(),\r
- objectInfo.Container, objectInfo.Name);\r
+ await client.MoveObject(objectInfo.Account, objectInfo.Container, oldName.Replace('\\','/').ToEscapedUri(),\r
+ objectInfo.Container, objectInfo.Name).ConfigureAwait(false);\r
+ StatusKeeper.MoveFileState(tuple.OldFullPath, tuple.FilePath, objectInfo, tuple.Merkle);\r
+ //StatusKeeper.StoreInfo(tuple.FilePath,objectInfo,tuple.Merkle);\r
+ //StatusKeeper.ClearFolderStatus(tuple.FilePath);\r
}\r
return true;\r
}\r
return serverPath;\r
}\r
\r
- using (StatusNotification.GetNotifier("Moving local {0}", "Moved local {0}", Path.GetFileName(tuple.FilePath)))\r
+ using (StatusNotification.GetNotifier("Moving local {0}", "Moved local {0}", true,Path.GetFileName(tuple.FilePath)))\r
using(NetworkGate.Acquire(tuple.FilePath,NetworkOperation.Renaming))\r
{\r
\r
\r
private void DeleteCloudFile(AccountInfo accountInfo, StateTuple tuple)\r
{\r
- using (StatusNotification.GetNotifier("Deleting server {0}", "Deleted server {0}", Path.GetFileName(tuple.FilePath)))\r
+ using (StatusNotification.GetNotifier("Deleting server {0}", "Deleted server {0}", true,Path.GetFileName(tuple.FilePath)))\r
{\r
\r
StatusKeeper.SetFileState(tuple.FilePath, FileStatus.Deleted,\r
}\r
\r
\r
- /*\r
- * //Use the queue to retry locked file hashing\r
- var fileQueue = new ConcurrentQueue<FileSystemInfo>(localInfos);\r
- \r
\r
- var results = new List<Tuple<FileSystemInfo, string>>();\r
- var backoff = 0;\r
- while (fileQueue.Count > 0)\r
- {\r
- FileSystemInfo file;\r
- fileQueue.TryDequeue(out file);\r
- using (ThreadContext.Stacks["File"].Push(file.FullName))\r
- {\r
- try\r
- {\r
- //Replace MD5 here, do the calc while syncing individual files\r
- string hash ;\r
- if (file is DirectoryInfo)\r
- hash = MD5_EMPTY;\r
- else\r
- {\r
- //Wait in case the FileAgent has requested a Pause\r
- await _unPauseEvent.WaitAsync().ConfigureAwait(false);\r
- \r
- using (StatusNotification.GetNotifier("Hashing {0}", "", file.Name))\r
- {\r
- hash = ((FileInfo)file).ComputeShortHash(StatusNotification);\r
- backoff = 0;\r
- }\r
- } \r
- results.Add(Tuple.Create(file, hash));\r
- }\r
- catch (IOException exc)\r
- {\r
- Log.WarnFormat("[HASH] File in use, will retry [{0}]", exc);\r
- fileQueue.Enqueue(file);\r
- //If this is the only enqueued file \r
- if (fileQueue.Count != 1) continue;\r
- \r
- \r
- //Increase delay\r
- if (backoff<60000)\r
- backoff += 10000;\r
- //Pause Polling for the specified time\r
- }\r
- if (backoff>0)\r
- await PauseFor(backoff).ConfigureAwait(false);\r
- }\r
- }\r
-\r
- return results;\r
-\r
- */\r
- private IEnumerable<StateTuple> MergeSources(IEnumerable<Tuple<string, ObjectInfo>> infos, IEnumerable<FileSystemInfo> files, List<FileState> states, ConcurrentDictionary<string, MovedEventArgs> moves)\r
- {\r
- var tuplesByPath = new Dictionary<string, StateTuple>();\r
- foreach (var info in files)\r
- {\r
- var tuple = new StateTuple(info);\r
- //Is this the target of a move event?\r
- var moveArg = moves.Values.FirstOrDefault(arg => info.FullName.Equals(arg.FullPath, StringComparison.InvariantCultureIgnoreCase) \r
- || info.FullName.IsAtOrBelow(arg.FullPath));\r
- if (moveArg != null)\r
- {\r
- tuple.NewFullPath = info.FullName;\r
- var relativePath = info.AsRelativeTo(moveArg.FullPath); \r
- tuple.OldFullPath = Path.Combine(moveArg.OldFullPath, relativePath);\r
- tuple.OldChecksum = states.FirstOrDefault(st => st.FilePath.Equals(tuple.OldFullPath, StringComparison.InvariantCultureIgnoreCase))\r
- .NullSafe(st => st.Checksum);\r
- }\r
-\r
- tuplesByPath[tuple.FilePath] = tuple;\r
- }\r
- \r
-\r
- \r
- \r
- //For files that have state\r
- foreach (var state in states)\r
- {\r
- StateTuple hashTuple;\r
-\r
- \r
- if (tuplesByPath.TryGetValue(state.FilePath, out hashTuple))\r
- {\r
- hashTuple.FileState = state;\r
- UpdateHashes(hashTuple);\r
- }\r
- else if (moves.ContainsKey(state.FilePath) && tuplesByPath.TryGetValue(moves[state.FilePath].FullPath, out hashTuple))\r
- {\r
- hashTuple.FileState = state;\r
- UpdateHashes(hashTuple);\r
- }\r
- else\r
- {\r
- var fsInfo = FileInfoExtensions.FromPath(state.FilePath);\r
- hashTuple = new StateTuple {FileInfo = fsInfo, FileState = state};\r
-\r
- //Is the source of a moved item?\r
- var moveArg = moves.Values.FirstOrDefault(arg => state.FilePath.Equals(arg.OldFullPath,StringComparison.InvariantCultureIgnoreCase) \r
- || state.FilePath.IsAtOrBelow(arg.OldFullPath));\r
- if (moveArg != null)\r
- {\r
- var relativePath = state.FilePath.AsRelativeTo(moveArg.OldFullPath);\r
- hashTuple.NewFullPath = Path.Combine(moveArg.FullPath,relativePath);\r
- hashTuple.OldFullPath = state.FilePath;\r
- //Do we have the old MD5?\r
- //hashTuple.OldMD5 = state.LastMD5;\r
- }\r
-\r
-\r
- tuplesByPath[state.FilePath] = hashTuple;\r
- }\r
- }\r
- //for files that don't have state\r
- foreach (var tuple in tuplesByPath.Values.Where(t => t.FileState == null))\r
- {\r
- UpdateHashes(tuple);\r
- }\r
-\r
- var tuplesByID = tuplesByPath.Values\r
- .Where(tuple => tuple.FileState != null && tuple.FileState.ObjectID!=null)\r
- .ToDictionary(tuple=>tuple.FileState.ObjectID,tuple=>tuple);//new Dictionary<Guid, StateTuple>();\r
-\r
- foreach (var info in infos)\r
- {\r
- StateTuple hashTuple;\r
- var filePath = info.Item1;\r
- var objectInfo = info.Item2;\r
- var objectID = objectInfo.UUID;\r
-\r
- if (objectID != _emptyGuid && tuplesByID.TryGetValue(objectID, out hashTuple))\r
- {\r
- hashTuple.ObjectInfo = objectInfo; \r
- }\r
- else if (tuplesByPath.TryGetValue(filePath, out hashTuple))\r
- {\r
- hashTuple.ObjectInfo = objectInfo;\r
- }\r
- else\r
- {\r
-\r
- \r
- var fsInfo = FileInfoExtensions.FromPath(filePath);\r
- hashTuple= new StateTuple {FileInfo = fsInfo, ObjectInfo = objectInfo};\r
- tuplesByPath[filePath] = hashTuple;\r
- \r
- if (objectInfo.UUID!=_emptyGuid)\r
- tuplesByID[objectInfo.UUID] = hashTuple;\r
- }\r
- }\r
-\r
- var tuples = tuplesByPath.Values;\r
- var brokenTuples = from tuple in tuples\r
- where tuple.FileState != null && tuple.FileState.Checksum == null\r
- && tuple.ObjectInfo != null && (tuple.FileInfo==null || !tuple.FileInfo.Exists)\r
- select tuple;\r
- var actualTuples = tuples.Except(brokenTuples);\r
- Debug.Assert(actualTuples.All(t => t.HashesValid()));\r
-\r
- foreach (var tuple in brokenTuples)\r
- {\r
- StatusKeeper.SetFileState(tuple.FileState.FilePath, \r
- FileStatus.Conflict, FileOverlayStatus.Conflict, "FileState without checksum encountered for server object missing from disk"); \r
- }\r
- \r
- return actualTuples;\r
- }\r
\r
\r
- /// <summary>\r
- /// Update the tuple with the file's hashes, avoiding calculation if the file is unchanged\r
- /// </summary>\r
- /// <param name="hashTuple"></param>\r
- /// <remarks>\r
- /// The function first checks the file's size and last write date to see if there are any changes. If there are none,\r
- /// the file's stored hashes are used.\r
- /// Otherwise, MD5 is calculated first to ensure there are no changes. If MD5 is different, the Merkle hash is calculated\r
- /// </remarks>\r
- private void UpdateHashes(StateTuple hashTuple)\r
- {\r
- \r
- try\r
- {\r
- var state = hashTuple.NullSafe(s => s.FileState);\r
- var storedHash = state.NullSafe(s => s.Checksum);\r
- var storedHashes = state.NullSafe(s => s.Hashes);\r
- //var storedMD5 = state.NullSafe(s => s.LastMD5);\r
- var storedDate = state.NullSafe(s => s.LastWriteDate) ?? DateTime.MinValue;\r
- var storedLength = state.NullSafe(s => s.LastLength);\r
\r
- //var md5Hash = Signature.MD5_EMPTY; \r
- var merkle=TreeHash.Empty;\r
-\r
- if (hashTuple.FileInfo is FileInfo)\r
- {\r
- var file = (FileInfo)hashTuple.FileInfo.WithProperCapitalization();\r
- \r
- //Attributes unchanged?\r
- //LastWriteTime is only accurate to the second\r
- var unchangedAttributes = file.LastWriteTime - storedDate < TimeSpan.FromSeconds(1) \r
- && storedLength == file.Length;\r
- \r
- //Attributes appear unchanged but the file length doesn't match the stored hash ?\r
- var nonEmptyMismatch = unchangedAttributes && \r
- (file.Length == 0 ^ storedHash== Signature.MERKLE_EMPTY);\r
-\r
- //Missing hashes for NON-EMPTY hash ?\r
- var missingHashes = storedHash != Signature.MERKLE_EMPTY &&\r
- String.IsNullOrWhiteSpace(storedHashes);\r
-\r
- //Unchanged attributes but changed MD5 \r
- //Short-circuiting ensures MD5 is computed only if the attributes are changed\r
- \r
- //var md5Mismatch = (!unchangedAttributes && file.ComputeShortHash(StatusNotification) != storedMD5);\r
-\r
-\r
- //If the attributes are unchanged but the Merkle doesn't match the size,\r
- //or the attributes and the MD5 hash have changed, \r
- //or the hashes are missing but the tophash is NOT empty, we need to recalculate\r
- //\r
- //Otherwise we load the hashes from state\r
- if (!unchangedAttributes || nonEmptyMismatch || missingHashes)\r
- merkle = RecalculateTreehash(file);\r
- else\r
- {\r
- merkle=TreeHash.Parse(hashTuple.FileState.Hashes);\r
- //merkle.MD5 = storedMD5;\r
- }\r
-\r
-\r
- //md5Hash = merkle.MD5;\r
- }\r
- //hashTuple.MD5 = md5Hash;\r
- //Setting Merkle also updates C\r
- hashTuple.Merkle = merkle;\r
- }\r
- catch (IOException)\r
- {\r
- hashTuple.Locked = true;\r
- } \r
- }\r
-\r
- /// <summary>\r
- /// Recalculate a file's treehash and md5 and update the database\r
- /// </summary>\r
- /// <param name="file"></param>\r
- /// <returns></returns>\r
- private TreeHash RecalculateTreehash(FileInfo file)\r
- {\r
- var progress = new Progress<double>(d =>StatusNotification.Notify(\r
- new StatusNotification(String.Format("Hashing {0} of {1}", d, file.Name))));\r
- var merkle = Signature.CalculateTreeHash(file, StatusKeeper.BlockSize, \r
- StatusKeeper.BlockHash,CancellationToken, progress);\r
- StatusKeeper.UpdateFileTreeHash(file.FullName, merkle);\r
- return merkle;\r
- }\r
\r
/// <summary>\r
/// Returns the latest LastModified date from the list of objects, but only if it is before\r
/// <param name="threshold"></param>\r
/// <param name="cloudObjects"></param>\r
/// <returns></returns>\r
- private static DateTime? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
+ private static DateTimeOffset? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
{\r
- DateTime? maxDate = null;\r
+ DateTimeOffset? maxDate = null;\r
if (cloudObjects!=null && cloudObjects.Count > 0)\r
maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
- if (maxDate == null || maxDate == DateTime.MinValue)\r
+ if (!maxDate.HasValue)\r
return threshold;\r
- if (threshold == null || threshold == DateTime.MinValue || threshold > maxDate)\r
+ if (!threshold.HasValue|| threshold > maxDate)\r
return maxDate;\r
return threshold;\r
}\r
/// <param name="threshold"></param>\r
/// <param name="cloudObjects"></param>\r
/// <returns></returns>\r
- private static DateTime? GetLatestDateAfter(DateTime? threshold, IList<ObjectInfo> cloudObjects)\r
+ private static DateTimeOffset? GetLatestDateAfter(DateTimeOffset? threshold, IList<ObjectInfo> cloudObjects)\r
{\r
- DateTime? maxDate = null;\r
+ DateTimeOffset? maxDate = null;\r
if (cloudObjects!=null && cloudObjects.Count > 0)\r
maxDate = cloudObjects.Max(obj => obj.Last_Modified);\r
- if (maxDate == null || maxDate == DateTime.MinValue)\r
+ if (!maxDate.HasValue)\r
return threshold;\r
- if (threshold == null || threshold == DateTime.MinValue || threshold < maxDate)\r
+ if (!threshold.HasValue|| threshold < maxDate)\r
return maxDate;\r
return threshold;\r
}\r
\r
readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
private bool _pause;\r
- private readonly string _emptyGuid = Guid.Empty.ToString();\r
+ \r
\r
\r
\r
\r
public void RemoveAccount(AccountInfo accountInfo)\r
{\r
+ if (accountInfo == null)\r
+ return;\r
+\r
AccountInfo account;\r
_accounts.TryRemove(accountInfo.AccountKey, out account);\r
\r
projects / pithos-ms-client / blobdiff