Synnefo » Pithos

from os.path import exists, expanduser, isdir, isfile, join, split

from shutil import copyfile

from time import time

from lib.transfer import download, upload

from lib.hashmap import merkle

SETTINGS_DIR = expanduser('~/.pithos')

TRASH_DIR = '.pithos_trash'

SQL_CREATE_FILES_TABLE = '''CREATE TABLE IF NOT EXISTS files (

                                path TEXT PRIMARY KEY,

                                hash TEXT,

                                timestamp INTEGER)'''

client = Pithos_Client(get_server(), get_auth(), get_user())

def _makedirs(path):

    try:

        os.makedirs(path)

    except OSError:

        pass

class State(object):

    def __init__(self, syncdir, container):

        self.syncdir = syncdir

        self.container = container

        self.trashdir = join(syncdir, TRASH_DIR)

        self.deleted_dirs = set()

        _makedirs(self.trashdir)

        dbpath = join(SETTINGS_DIR, 'sync.db')

        self.conn.execute(SQL_CREATE_FILES_TABLE)

    def current_hash(self, path):

        """Return the hash of the file as it exists now in the filesystem"""

        fullpath = join(self.syncdir, path)

        if fullpath in self.deleted_dirs:

            return 'DEL'

        if not exists(fullpath):

            return 'DEL'

        if isdir(fullpath):

            return 'DIR'

        return merkle(fullpath)

    def delete_inactive(self, timestamp):

        sql = 'DELETE FROM files WHERE timestamp != ?'

        self.conn.execute(sql, (timestamp,))

    def download(self, path, hash):

        fullpath = join(self.syncdir, path)

        if hash == 'DEL':

            self.trash(path)

        elif hash == 'DIR':

            _makedirs(fullpath)

        else:

            self.trash(path)    # Trash any old version

            localpath = self.find_hash(hash)

            if localpath:

                copyfile(localpath, fullpath)

            else:

                print 'Downloading %s...' % path

                download(client, self.container, path, fullpath)

        current = self.current_hash(path)

        assert current == hash, "Downloaded file does not match hash"

        self.save(path, hash)

    def empty_trash(self):

        for filename in os.listdir(self.trashdir):

            path = join(self.trashdir, filename)

            os.remove(path)

    def find_hash(self, hash):

        if ret:

            return join(self.syncdir, ret[0])

        if hash in os.listdir(self.trashdir):

            return join(self.trashdir, hash)

        return None

    def previous_hash(self, path):

        """Return the hash of the file according to the previous sync with

           the server. Return DEL if not such entry exists."""

        sql = 'SELECT hash FROM files WHERE path = ?'

        ret = self.conn.execute(sql, (path,)).fetchone()

        return ret[0] if ret else 'DEL'

    def remote_hash(self, path):

        """Return the hash of the file according to the server"""

            meta = client.retrieve_object_metadata(self.container, path)

    def remove_deleted_dirs(self):

        for path in sorted(self.deleted_dirs, key=len, reverse=True):

            os.rmdir(path)

            self.deleted_dirs.remove(path)

    def resolve_conflict(self, path, hash):

        """Resolve a sync conflict by renaming the local file and downloading

           the remote one."""

        fullpath = join(self.syncdir, path)

        resolved = fullpath + '.local'

        i = 0

        while exists(resolved):

            i += 1

            resolved = fullpath + '.local%d' % i

        os.rename(fullpath, resolved)

        self.download(path, hash)

    def rmdir(self, path):

        """Remove a dir or mark for deletion if non-empty

        If a dir is empty delete it and check if any of its parents should be

        deleted too. Else mark it for later deletion.

        """

        fullpath = join(self.syncdir, path)

        if not exists(fullpath):

            return

        if os.listdir(fullpath):

            # Directory not empty

            self.deleted_dirs.add(fullpath)

            return

        os.rmdir(fullpath)

        self.deleted_dirs.discard(fullpath)

        parent = dirname(fullpath)

        while parent in self.deleted_dirs:

            os.rmdir(parent)

            self.deleted_dirs.remove(parent)

            parent = dirname(parent)

    def save(self, path, hash):

        """Save the hash value of a file. This value will be later returned

           by `previous_hash`."""

        sql = 'INSERT OR REPLACE INTO files (path, hash) VALUES (?, ?)'

        self.conn.execute(sql, (path, hash))

        self.conn.commit()

    def touch(self, path, now):

        sql = 'UPDATE files SET timestamp = ? WHERE path = ?'

        self.conn.execute(sql, (now, path))

        self.conn.commit()

    def trash(self, path):

        """Move a file to trash or delete it if it's a directory"""

        fullpath = join(self.syncdir, path)

        if not exists(fullpath):

            return

        if isfile(fullpath):

            hash = merkle(fullpath)

            trashpath = join(self.trashdir, hash)

            os.rename(fullpath, trashpath)

            self.rmdir(path)

    def upload(self, path, hash):

        fullpath = join(self.syncdir, path)

        if hash == 'DEL':

            client.delete_object(self.container, path)

        elif hash == 'DIR':

            client.create_directory_marker(self.container, path)

        else:

            prefix, name = split(path)

            if prefix:

                prefix += '/'

            print 'Uploading %s...' % path

            upload(client, fullpath, self.container, prefix, name)

        remote = self.remote_hash(path)

        assert remote == hash, "Uploaded file does not match hash"

        self.save(path, hash)

def sync(path, state):

    previous = state.previous_hash(path)

    current = state.current_hash(path)

    remote = state.remote_hash(path)

    if current == previous:

        # No local changes, download any remote changes

        if remote != previous:

            state.download(path, remote)

    elif remote == previous:

        # No remote changes, upload any local changes

        if current != previous:

            state.upload(path, current)

        # Both local and remote file have changes since last sync

        if current == remote:

            state.save(path, remote)    # Local and remote changes match

        else:

            state.resolve_conflict(path, remote)

def walk(dir, container):

    """Iterates on the files of the hierarchy created by merging the files

       in `dir` and the objects in `container`."""

        root = pending.pop(0)   # Depth First Traversal

        if root == TRASH_DIR:

            continue

        dirpath = join(dir, root)

        if exists(dirpath):

                path = join(root, filename)

                if isdir(join(dir, path)):

        for object in client.list_objects(container, format='json',

                prefix=root, delimiter='/'):

            name = object['name']

    syncdir = sys.argv[1]

    _makedirs(SETTINGS_DIR)

    container = os.environ.get('PITHOS_SYNC_CONTAINER', DEFAULT_CONTAINER)

    client.create_container(container)

    state = State(syncdir, container)

    now = int(time())

    for path in walk(syncdir, container):

        sync(path, state)

        state.touch(path, now)

    state.delete_inactive(now)

    state.empty_trash()

    state.remove_deleted_dirs()