}\r
\r
private FileSystemInfo _fileInfo;\r
+ private TreeHash _merkle;\r
+\r
public FileSystemInfo FileInfo\r
{\r
get { return _fileInfo; }\r
public FileState FileState { get; set; }\r
public ObjectInfo ObjectInfo{ get; set; }\r
\r
+\r
+ public TreeHash Merkle\r
+ {\r
+ get {\r
+ return _merkle;\r
+ }\r
+ set {\r
+ _merkle = value;\r
+ C = _merkle.TopHash.ToHashString();\r
+ }\r
+ }\r
+\r
public StateTuple() { }\r
\r
public StateTuple(FileSystemInfo info)\r
/// <summary>\r
/// Start a manual synchronization\r
/// </summary>\r
- public void SynchNow()\r
+ public void SynchNow(IEnumerable<string> paths=null)\r
{ \r
+ _batchQueue.Enqueue(paths);\r
_syncEvent.Set();\r
}\r
\r
+ readonly ConcurrentQueue<IEnumerable<string>> _batchQueue=new ConcurrentQueue<IEnumerable<string>>();\r
\r
/// <summary>\r
/// Remote files are polled periodically. Any changes are processed\r
await _unPauseEvent.WaitAsync();\r
UpdateStatus(PithosStatus.PollSyncing);\r
\r
- var tasks = from accountInfo in _accounts.Values\r
- select ProcessAccountFiles(accountInfo, since);\r
+ var accountBatches=new Dictionary<Uri, IEnumerable<string>>();\r
+ IEnumerable<string> batch = null;\r
+ if (_batchQueue.TryDequeue(out batch) && batch != null)\r
+ foreach (var account in _accounts.Values)\r
+ {\r
+ var accountBatch = batch.Where(path => path.IsAtOrBelow(account.AccountPath));\r
+ accountBatches[account.AccountKey] = accountBatch;\r
+ }\r
+\r
+\r
+ IEnumerable<Task<DateTime?>> tasks = new List<Task<DateTime?>>();\r
+ foreach(var accountInfo in _accounts.Values)\r
+ {\r
+ IEnumerable<string> accountBatch ;\r
+ accountBatches.TryGetValue(accountInfo.AccountKey,out accountBatch);\r
+ ProcessAccountFiles (accountInfo, accountBatch, since);\r
+ }\r
\r
var nextTimes=await TaskEx.WhenAll(tasks.ToList());\r
\r
private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since)\r
{\r
var sync = _syncEvent.WaitAsync();\r
- var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval), NetworkAgent.CancellationToken);\r
+ var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval));\r
\r
var signaledTask = await TaskEx.WhenAny(sync, wait);\r
\r
return since;\r
}\r
\r
- public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, DateTime? since = null)\r
+ public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, IEnumerable<string> accountBatch, DateTime? since = null)\r
{\r
if (accountInfo == null)\r
throw new ArgumentNullException("accountInfo");\r
\r
//var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes);\r
\r
- var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
+ //var filterUris = Selectives.SelectiveUris[accountInfo.AccountKey];\r
\r
\r
//Get the local files here \r
\r
var files = LoadLocalFileTuples(accountInfo);\r
\r
- var states = FileState.Queryable.ToList();\r
+ var states = FileState.Queryable.ToList(); \r
+ \r
\r
var infos = (from remote in cleanRemotes\r
let path = remote.RelativeUrlToFilePath(accountInfo.UserName)\r
\r
var tuples = MergeSources(infos, files, states).ToList();\r
\r
- \r
- foreach (var tuple in tuples)\r
+\r
+ var stateTuples = accountBatch==null?tuples:tuples.Where(t => accountBatch.Contains(t.FilePath));\r
+ foreach (var tuple in stateTuples)\r
{\r
await _unPauseEvent.WaitAsync();\r
\r
tuple.C = MERKLE_EMPTY;\r
else if (tuple.FileState != null && tuple.MD5 == tuple.FileState.ShortHash)\r
{\r
- //If there is a state whose MD5 matches, load the merkle hash fromthe file state\r
+ //If there is a state whose MD5 matches, load the merkle hash from the file state\r
//insteaf of calculating it\r
tuple.C = tuple.FileState.Checksum; \r
}\r
else\r
{\r
- tuple.C=Signature.CalculateTreeHash(tuple.FileInfo, accountInfo.BlockSize, accountInfo.BlockHash)\r
- .TopHash.ToHashString();\r
+ tuple.Merkle = Signature.CalculateTreeHash(tuple.FileInfo, accountInfo.BlockSize, accountInfo.BlockHash);\r
+ //tuple.C=tuple.Merkle.TopHash.ToHashString(); \r
}\r
}\r
\r
var localInfo = FileInfoExtensions.FromPath(localFilePath);\r
\r
\r
+ var isUnselectedRootFolder = agent.IsUnselectedRootFolder(tuple.FilePath);\r
+\r
+ //Unselected root folders that have not yet been uploaded should be uploaded and added to the \r
+ //selective folders\r
+\r
+ if (!Selectives.IsSelected(accountInfo, localFilePath) && !(isUnselectedRootFolder && tuple.ObjectInfo==null) ) \r
+ return;\r
+\r
// Local file unchanged? If both C and L are null, make sure it's because \r
//both the file is missing and the state checksum is not missing\r
- if (tuple.C == tuple.L && (localInfo.Exists || tuple.FileState==null))\r
+ if (tuple.C == tuple.L /*&& (localInfo.Exists || tuple.FileState == null)*/)\r
{\r
//No local changes\r
//Server unchanged?\r
else\r
{\r
//Different from server\r
- if (Selectives.IsSelected(accountInfo, localFilePath))\r
+ //Does the server file exist?\r
+ if (tuple.S == null)\r
{\r
- //Does the server file exist?\r
- if (tuple.S == null)\r
- {\r
- //Server file doesn't exist\r
- //deleteObjectFromLocal()\r
- StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
- FileOverlayStatus.Deleted, "");\r
- agent.Delete(localFilePath);\r
- //updateRecord(Remove C, L)\r
- StatusKeeper.ClearFileStatus(localFilePath);\r
- }\r
- else\r
- {\r
- //Server file exists\r
- //downloadServerObject() // Result: L = S\r
- //If the file has moved on the server, move it locally before downloading\r
- var targetPath=MoveForServerMove(accountInfo,tuple);\r
-\r
- StatusKeeper.SetFileState(targetPath, FileStatus.Modified,\r
- FileOverlayStatus.Modified, "");\r
- NetworkAgent.Downloader.DownloadCloudFile(accountInfo,\r
- tuple.ObjectInfo,\r
- targetPath, token).Wait(token);\r
- //updateRecord( L = S )\r
- StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag,\r
- tuple.ObjectInfo.X_Object_Hash);\r
-\r
- StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
+ //Server file doesn't exist\r
+ //deleteObjectFromLocal()\r
+ StatusKeeper.SetFileState(localFilePath, FileStatus.Deleted,\r
+ FileOverlayStatus.Deleted, "");\r
+ agent.Delete(localFilePath);\r
+ //updateRecord(Remove C, L)\r
+ StatusKeeper.ClearFileStatus(localFilePath);\r
+ }\r
+ else\r
+ {\r
+ //Server file exists\r
+ //downloadServerObject() // Result: L = S\r
+ //If the file has moved on the server, move it locally before downloading\r
+ var targetPath = MoveForServerMove(accountInfo, tuple);\r
+\r
+ StatusKeeper.SetFileState(targetPath, FileStatus.Modified,\r
+ FileOverlayStatus.Modified, "");\r
+ NetworkAgent.Downloader.DownloadCloudFile(accountInfo,\r
+ tuple.ObjectInfo,\r
+ targetPath, tuple.Merkle, token).Wait(token);\r
+ //updateRecord( L = S )\r
+ StatusKeeper.UpdateFileChecksum(targetPath, tuple.ObjectInfo.ETag,\r
+ tuple.ObjectInfo.X_Object_Hash);\r
+\r
+ StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
\r
-/*\r
- StatusKeeper.SetFileState(targetPath, FileStatus.Unchanged,\r
- FileOverlayStatus.Normal, "");\r
-*/\r
- }\r
+ /*\r
+ StatusKeeper.SetFileState(targetPath, FileStatus.Unchanged,\r
+ FileOverlayStatus.Normal, "");\r
+ */\r
}\r
}\r
+\r
}\r
else\r
{\r
else\r
{\r
//uploadLocalObject() // Result: S = C, L = S \r
- var isUnselected = agent.IsUnselectedRootFolder(tuple.FilePath);\r
\r
//Debug.Assert(tuple.FileState !=null);\r
var action = new CloudUploadAction(accountInfo, localInfo, tuple.FileState,\r
accountInfo.BlockSize, accountInfo.BlockHash,\r
- "Poll", isUnselected);\r
- NetworkAgent.Uploader.UploadCloudFile(action, token).Wait(token);\r
+ "Poll", isUnselectedRootFolder);\r
+ NetworkAgent.Uploader.UploadCloudFile(action, tuple.Merkle, token).Wait(token);\r
\r
//updateRecord( S = C )\r
//State updated by the uploader\r
- \r
- if (isUnselected)\r
+\r
+ if (isUnselectedRootFolder)\r
{\r
ProcessChildren(accountInfo, tuple, agent, token);\r
}\r
}\r
else\r
{\r
- if (Selectives.IsSelected(accountInfo, localFilePath))\r
+ if (tuple.C == tuple.S)\r
+ {\r
+ // (Identical Changes) Result: L = S\r
+ //doNothing()\r
+ //Detect server moves\r
+ var targetPath = MoveForServerMove(accountInfo, tuple);\r
+ StatusKeeper.StoreInfo(targetPath, tuple.ObjectInfo);\r
+ }\r
+ else\r
{\r
- if (tuple.C == tuple.S)\r
+ if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null)\r
+ {\r
+ //deleteObjectFromServer()\r
+ DeleteCloudFile(accountInfo, tuple);\r
+ //updateRecord(Remove L, S) \r
+ }\r
+ //If both the local and server files are missing, the state is stale\r
+ else if (!localInfo.Exists && (tuple.S == null || tuple.ObjectInfo == null))\r
{\r
- // (Identical Changes) Result: L = S\r
- //doNothing()\r
- //Detect server moves\r
- var targetPath=MoveForServerMove(accountInfo, tuple);\r
- StatusKeeper.StoreInfo(targetPath,tuple.ObjectInfo);\r
+ StatusKeeper.ClearFileStatus(localInfo.FullName);\r
}\r
else\r
{\r
- if ((tuple.C == null || !localInfo.Exists) && tuple.ObjectInfo != null )\r
- {\r
- //deleteObjectFromServer()\r
- DeleteCloudFile(accountInfo, tuple);\r
- //updateRecord(Remove L, S) \r
- }\r
- else\r
- {\r
- ReportConflictForMismatch(localFilePath);\r
- //identifyAsConflict() // Manual action required\r
- }\r
+ ReportConflictForMismatch(localFilePath);\r
+ //identifyAsConflict() // Manual action required\r
}\r
}\r
}\r
\r
private string MoveForServerMove(AccountInfo accountInfo, StateTuple tuple)\r
{\r
+ if (tuple.ObjectInfo == null)\r
+ return null;\r
var relativePath = tuple.ObjectInfo.RelativeUrlToFilePath(accountInfo.UserName);\r
var serverPath = Path.Combine(accountInfo.AccountPath, relativePath);\r
-\r
- if (tuple.FilePath == serverPath) return serverPath;\r
+ \r
+ //Compare Case Insensitive\r
+ if (String.Equals(tuple.FilePath ,serverPath,StringComparison.InvariantCultureIgnoreCase)) return serverPath;\r
\r
if (tuple.FileInfo.Exists)\r
{ \r
return threshold;\r
}\r
\r
- //readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
+ readonly AccountsDifferencer _differencer = new AccountsDifferencer();\r
private Dictionary<Uri, List<Uri>> _selectiveUris = new Dictionary<Uri, List<Uri>>();\r
private bool _pause;\r
private static string MERKLE_EMPTY = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";\r
{\r
AccountInfo account;\r
_accounts.TryRemove(accountInfo.AccountKey, out account);\r
-/*\r
+\r
SnapshotDifferencer differencer;\r
_differencer.Differencers.TryRemove(accountInfo.AccountKey, out differencer);\r
-*/\r
}\r
\r
public void SetSelectivePaths(AccountInfo accountInfo,Uri[] added, Uri[] removed)\r
{\r
AbortRemovedPaths(accountInfo,removed);\r
- DownloadNewPaths(accountInfo,added);\r
+ //DownloadNewPaths(accountInfo,added);\r
}\r
\r
+/*\r
private void DownloadNewPaths(AccountInfo accountInfo, Uri[] added)\r
{\r
var client = new CloudFilesClient(accountInfo);\r
//Need to get a listing of each of the URLs, then post them to the NetworkAgent\r
//CreatesToActions(accountInfo,)\r
\r
-/* NetworkAgent.Post();*/\r
+/* NetworkAgent.Post();#1#\r
}\r
+*/\r
\r
private void AbortRemovedPaths(AccountInfo accountInfo, Uri[] removed)\r
{\r
projects / pithos-ms-client / blobdiff