[System.ComponentModel.Composition.Import]
public IPithosSettings Settings { get; set; }
+ [System.ComponentModel.Composition.Import]
+ public IStatusNotification StatusNotification { get; set; }
+
private Agent<Action> _persistenceAgent;
private static void MigrateOldDb(string dbPath, string appDataPath)
{
- Contract.Requires(!String.IsNullOrWhiteSpace(dbPath));
- Contract.Requires(!String.IsNullOrWhiteSpace(appDataPath));
-
+ if(String.IsNullOrWhiteSpace(dbPath))
+ throw new ArgumentNullException("dbPath");
+ if(String.IsNullOrWhiteSpace(appDataPath))
+ throw new ArgumentNullException("appDataPath");
+ Contract.EndContractBlock();
var oldDbPath = Path.Combine(appDataPath, "Pithos", "pithos.db");
var oldDbInfo = new FileInfo(oldDbPath);
{
_persistenceAgent.Stop();
}
-
+
public void ProcessExistingFiles(IEnumerable<FileInfo> existingFiles)
{
- if(existingFiles ==null)
+ if (existingFiles == null)
throw new ArgumentNullException("existingFiles");
Contract.EndContractBlock();
-
+
//Find new or matching files with a left join to the stored states
- var fileStates = FileState.Queryable;
- var currentFiles=from file in existingFiles
- join state in fileStates on file.FullName.ToLower() equals state.FilePath.ToLower() into gs
- from substate in gs.DefaultIfEmpty()
- select new {File = file, State = substate};
+ var fileStates = FileState.Queryable.ToList();
+ var currentFiles = from file in existingFiles
+ join state in fileStates on file.FullName.ToLower() equals state.FilePath.ToLower() into
+ gs
+ from substate in gs.DefaultIfEmpty()
+ select Tuple.Create(file, substate);
//To get the deleted files we must get the states that have no corresponding
//files.
//We can't use the File.Exists method inside a query, so we get all file paths from the states
var statePaths = (from state in fileStates
- select new {state.Id, state.FilePath}).ToList();
+ select new {state.Id, state.FilePath}).ToList();
//and check each one
- var missingStates= (from path in statePaths
- where !File.Exists(path.FilePath) && !Directory.Exists(path.FilePath)
- select path.Id).ToList();
+ var missingStates = (from path in statePaths
+ where !File.Exists(path.FilePath) && !Directory.Exists(path.FilePath)
+ select path.Id).ToList();
//Finally, retrieve the states that correspond to the deleted files
- var deletedFiles = from state in fileStates
- where missingStates.Contains(state.Id)
- select new { File = default(FileInfo), State = state };
+ var deletedFiles = from state in fileStates
+ where missingStates.Contains(state.Id)
+ select Tuple.Create(default(FileInfo), state);
var pairs = currentFiles.Union(deletedFiles).ToList();
- using (var shortHasher = HashAlgorithm.Create("sha1"))
+ i = 1;
+ var total = pairs.Count;
+ foreach (var pair in pairs)
+ {
+ ProcessFile(total, pair);
+ }
+ }
+
+ int i = 1;
+
+ private void ProcessFile(int total, Tuple<FileInfo,FileState> pair)
+ {
+ var idx = Interlocked.Increment(ref i);
+ using (StatusNotification.GetNotifier("Indexing file {0} of {1}", "Indexed file {0} of {1} ", idx, total))
{
- foreach (var pair in pairs)
+ var fileState = pair.Item2;
+ var file = pair.Item1;
+ if (fileState == null)
{
- var fileState = pair.State;
- var file = pair.File;
- if (fileState == null)
- {
- //This is a new file
- var createState = FileState.CreateFor(file);
- _persistenceAgent.Post(createState.Create);
- }
- else if (file == null)
- {
- //This file was deleted while we were down. We should mark it as deleted
- //We have to go through UpdateStatus here because the state object we are using
- //was created by a different ORM session.
- _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Deleted));
- }
- else
+ //This is a new file
+ var createState = FileState.CreateFor(file,StatusNotification);
+ _persistenceAgent.Post(createState.Create);
+ }
+ else if (file == null)
+ {
+ //This file was deleted while we were down. We should mark it as deleted
+ //We have to go through UpdateStatus here because the state object we are using
+ //was created by a different ORM session.
+ _persistenceAgent.Post(() => UpdateStatusDirect((Guid) fileState.Id, FileStatus.Deleted));
+ }
+ else
+ {
+ //This file has a matching state. Need to check for possible changes
+ //To check for changes, we use the cheap (in CPU terms) MD5 algorithm
+ //on the entire file.
+
+ var hashString = file.ComputeShortHash(StatusNotification);
+ Debug.Assert(hashString.Length==32);
+
+
+ //TODO: Need a way to attach the hashes to the filestate so we don't
+ //recalculate them each time a call to calculate has is made
+ //We can either store them to the filestate or add them to a
+ //dictionary
+
+ //If the hashes don't match the file was changed
+ if (fileState.ETag != hashString)
{
- //This file has a matching state. Need to check for possible changes
- //To check for changes, we use the cheap (in CPU terms) SHA1 algorithm
- //on the entire file.
-
- var hashString = file.ComputeShortHash(shortHasher);
- //TODO: Need a way to attach the hashes to the filestate so we don't
- //recalculate them each time a call to calculate has is made
- //We can either store them to the filestate or add them to a
- //dictionary
-
- //If the hashes don't match the file was changed
- if (fileState.ShortHash != hashString)
- {
- _persistenceAgent.Post(() => UpdateStatusDirect(fileState.Id, FileStatus.Modified));
- }
+ _persistenceAgent.Post(() => UpdateStatusDirect((Guid) fileState.Id, FileStatus.Modified));
}
}
}
-
-
}
-
private int UpdateStatusDirect(Guid id, FileStatus status)
command.Parameters.AddWithValue("path", path);
var affected = command.ExecuteNonQuery();
+ if (affected == 0)
+ {
+ var createdState = FileState.CreateFor(FileInfoExtensions.FromPath(path), StatusNotification);
+ createdState.FileStatus = status;
+ createdState.Create();
+ }
return affected;
}
}
}
}
- private int UpdateStatusDirect(string absolutePath, FileStatus fileStatus, FileOverlayStatus overlayStatus)
+ private int UpdateStatusDirect(string absolutePath, FileStatus fileStatus, FileOverlayStatus overlayStatus, string conflictReason)
{
using (log4net.ThreadContext.Stacks["StatusAgent"].Push("UpdateStatusDirect"))
{
using (
var command =
new SQLiteCommand(
- "update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE ",
+ "update FileState set OverlayStatus= :overlayStatus, FileStatus= :fileStatus,ConflictReason= :conflictReason where FilePath = :path COLLATE NOCASE ",
connection))
{
command.Parameters.AddWithValue("path", absolutePath);
command.Parameters.AddWithValue("fileStatus", fileStatus);
command.Parameters.AddWithValue("overlayStatus", overlayStatus);
+ command.Parameters.AddWithValue("conflictReason", conflictReason);
var affected = command.ExecuteNonQuery();
+ if (affected == 0)
+ {
+ var createdState = FileState.CreateFor(FileInfoExtensions.FromPath(absolutePath), StatusNotification);
+ createdState.FileStatus = fileStatus;
+ createdState.OverlayStatus = overlayStatus;
+ createdState.ConflictReason = conflictReason;
+ createdState.LastMD5 = String.Empty;
+ createdState.Create();
+ }
return affected;
}
}
{
using (var connection = GetConnection())
- using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,ShortHash,Version ,VersionTimeStamp,IsShared ,SharedBy ,ShareWrite from FileState where FilePath=:path COLLATE NOCASE", connection))
+ using (var command = new SQLiteCommand("select Id, FilePath, OverlayStatus,FileStatus ,Checksum ,ETag,Version ,VersionTimeStamp,IsShared ,SharedBy ,ShareWrite from FileState where FilePath=:path COLLATE NOCASE", connection))
{
command.Parameters.AddWithValue("path", path);
OverlayStatus =reader.IsDBNull(2)?FileOverlayStatus.Unversioned: (FileOverlayStatus) reader.GetInt64(2),
FileStatus = reader.IsDBNull(3)?FileStatus.Missing:(FileStatus) reader.GetInt64(3),
Checksum = reader.IsDBNull(4)?"":reader.GetString(4),
- ShortHash= reader.IsDBNull(5)?"":reader.GetString(5),
+ ETag= reader.IsDBNull(5)?"":reader.GetString(5),
Version = reader.IsDBNull(6)?default(long):reader.GetInt64(6),
VersionTimeStamp = reader.IsDBNull(7)?default(DateTime):reader.GetDateTime(7),
IsShared = !reader.IsDBNull(8) && reader.GetBoolean(8),
_persistenceAgent.Post(() => FileState.StoreOverlayStatus(path,overlayStatus));
}*/
- public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string shortHash = null)
+ public Task SetFileOverlayStatus(string path, FileOverlayStatus overlayStatus, string etag = null)
{
if (String.IsNullOrWhiteSpace(path))
throw new ArgumentNullException("path");
throw new ArgumentException("The path must be rooted","path");
Contract.EndContractBlock();
- return _persistenceAgent.PostAndAwait(() => FileState.StoreOverlayStatus(path,overlayStatus,shortHash));
+ return _persistenceAgent.PostAndAwait(() => FileState.StoreOverlayStatus(path,overlayStatus,etag));
}
/* public void RenameFileOverlayStatus(string oldPath, string newPath)
_persistenceAgent.Post(() =>FileState.RenameState(oldPath, newPath));
}*/
- public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus)
+ public void SetFileState(string path, FileStatus fileStatus, FileOverlayStatus overlayStatus, string conflictReason)
{
if (String.IsNullOrWhiteSpace(path))
throw new ArgumentNullException("path");
Debug.Assert(!path.Contains(FolderConstants.CacheFolder));
Debug.Assert(!path.EndsWith(".ignore"));
- _persistenceAgent.Post(() => UpdateStatusDirect(path, fileStatus, overlayStatus));
+ _persistenceAgent.Post(() => UpdateStatusDirect(path, fileStatus, overlayStatus, conflictReason));
}
/*
{
using (var connection = GetConnection())
+ using(var tx=connection.BeginTransaction())
using (var command = new SQLiteCommand(connection))
{
- if (StateExists(path, connection))
+ //If the ID exists, update the status
+ command.CommandText ="update FileState set FilePath=:path,FileStatus= :fileStatus, Checksum=:checksum, ETag=:etag,LastMD5=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp where ObjectID = :objectID ";
+ command.Parameters.AddWithValue("path", path);
+ command.Parameters.AddWithValue("checksum", objectInfo.X_Object_Hash);
+ command.Parameters.AddWithValue("etag", objectInfo.ETag);
+ command.Parameters.AddWithValue("version", objectInfo.Version);
+ command.Parameters.AddWithValue("versionTimeStamp", objectInfo.VersionTimestamp);
+ command.Parameters.AddWithValue("fileStatus", FileStatus.Unchanged);
+ command.Parameters.AddWithValue("overlayStatus", FileOverlayStatus.Normal);
+ command.Parameters.AddWithValue("objectID",objectInfo.UUID);
+ var affected = command.ExecuteNonQuery();
+ if (affected == 0)
+ {
+ //If the ID doesn't exist, try to update using the path, and store the ID as well.
command.CommandText =
- "update FileState set FileStatus= :fileStatus where FilePath = :path COLLATE NOCASE ";
- else
+ "update FileState set FileStatus= :fileStatus, ObjectID=:objectID, Checksum=:checksum, ETag=:etag,LastMD5=:etag,Version=:version,VersionTimeStamp=:versionTimeStamp where FilePath = :path COLLATE NOCASE ";
+ affected = command.ExecuteNonQuery();
+ }
+ if (affected==0)
{
command.CommandText =
- "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,ShortHash,FileStatus,OverlayStatus) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:shortHash,:fileStatus,:overlayStatus)";
+ "INSERT INTO FileState (Id,FilePath,Checksum,Version,VersionTimeStamp,ETag,LastMD5,FileStatus,OverlayStatus,ObjectID) VALUES (:id,:path,:checksum,:version,:versionTimeStamp,:etag,:etag,:fileStatus,:overlayStatus,:objectID)";
command.Parameters.AddWithValue("id", Guid.NewGuid());
+ affected = command.ExecuteNonQuery();
}
- command.Parameters.AddWithValue("path", path);
- command.Parameters.AddWithValue("checksum", objectInfo.Hash);
- command.Parameters.AddWithValue("shortHash", "");
- command.Parameters.AddWithValue("version", objectInfo.Version);
- command.Parameters.AddWithValue("versionTimeStamp",
- objectInfo.VersionTimestamp);
- command.Parameters.AddWithValue("fileStatus", FileStatus.Unchanged);
- command.Parameters.AddWithValue("overlayStatus",
- FileOverlayStatus.Normal);
-
- var affected = command.ExecuteNonQuery();
- return;
}
}
catch (Exception exc)
{
- Log.Error(exc.ToString());
+ Log.ErrorFormat("Failed to update [{0}]:[{1}]\r\n{2}",path,objectInfo.UUID, exc);
throw;
}
}
}
+ private bool StateExistsByID(string objectId,SQLiteConnection connection)
+ {
+ using (var command = new SQLiteCommand("Select count(*) from FileState where ObjectId=:id", connection))
+ {
+ command.Parameters.AddWithValue("id", objectId);
+ var result = command.ExecuteScalar();
+ return ((long)result >= 1);
+ }
+
+ }
+
public void SetFileStatus(string path, FileStatus status)
{
if (String.IsNullOrWhiteSpace(path))
if (!Path.IsPathRooted(path))
throw new ArgumentException("The path must be rooted", "path");
Contract.EndContractBlock();
-
+ //TODO: May throw if the agent is cleared for some reason. Should never happen
_persistenceAgent.Post(() => DeleteFolderDirect(path));
}
var fileInfo = FileInfoExtensions.FromPath(path);
using (new SessionScope())
{
- var newState = FileState.CreateFor(fileInfo);
+ var newState = FileState.CreateFor(fileInfo,StatusNotification);
newState.FileStatus=FileStatus.Missing;
_persistenceAgent.PostAndAwait(newState.CreateAndFlush).Wait();
}
}
}
- public void UpdateFileChecksum(string path, string shortHash, string checksum)
+ public void UpdateFileChecksum(string path, string etag, string checksum)
{
if (String.IsNullOrWhiteSpace(path))
throw new ArgumentNullException("path");
throw new ArgumentException("The path must be rooted", "path");
Contract.EndContractBlock();
- _persistenceAgent.Post(() => FileState.UpdateChecksum(path, shortHash,checksum));
+ _persistenceAgent.Post(() => FileState.UpdateChecksum(path, etag,checksum));
+ }
+
+ public void UpdateLastMD5(FileInfo file, string etag)
+ {
+ if (file==null)
+ throw new ArgumentNullException("file");
+ if (String.IsNullOrWhiteSpace(etag))
+ throw new ArgumentNullException("etag");
+ Contract.EndContractBlock();
+
+ _persistenceAgent.Post(() => FileState.UpdateLastMD5(file, etag));
+ }
+
+
+ public void CleanupOrphanStates()
+ {
+ //Orphan states are those that do not correspond to an account, ie. their paths
+ //do not start with the root path of any registered account
+
+ var roots=(from account in Settings.Accounts
+ select account.RootPath).ToList();
+
+ var allStates = from state in FileState.Queryable
+ select state.FilePath;
+
+ foreach (var statePath in allStates)
+ {
+ if (!roots.Any(root=>statePath.StartsWith(root,StringComparison.InvariantCultureIgnoreCase)))
+ this.DeleteDirect(statePath);
+ }
}
+ public void CleanupStaleStates(AccountInfo accountInfo, List<ObjectInfo> objectInfos)
+ {
+ if (accountInfo == null)
+ throw new ArgumentNullException("accountInfo");
+ if (objectInfos == null)
+ throw new ArgumentNullException("objectInfos");
+ Contract.EndContractBlock();
+
+
+
+ //Stale states are those that have no corresponding local or server file
+
+
+ var agent=FileAgent.GetFileAgent(accountInfo);
+
+ var localFiles=agent.EnumerateFiles();
+ var localSet = new HashSet<string>(localFiles);
+
+ //RelativeUrlToFilePath will fail for
+ //infos of accounts, containers which have no Name
+
+ var serverFiles = from info in objectInfos
+ where info.Name != null
+ select Path.Combine(accountInfo.AccountPath,info.RelativeUrlToFilePath(accountInfo.UserName));
+ var serverSet = new HashSet<string>(serverFiles);
+
+ var allStates = from state in FileState.Queryable
+ where state.FilePath.StartsWith(agent.RootPath)
+ select state.FilePath;
+ var stateSet = new HashSet<string>(allStates);
+ stateSet.ExceptWith(serverSet);
+ stateSet.ExceptWith(localSet);
+
+ foreach (var remainder in stateSet)
+ {
+ DeleteDirect(remainder);
+ }
+
+
+ }
}
projects / pithos-ms-client / blobdiff