Synnefo

#!/usr/bin/env python

# Copyright 2011-2012 GRNET S.A. All rights reserved.

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

        self.deleted_dirs = set()

        _makedirs(self.trashdir)

        dbpath = join(SETTINGS_DIR, 'sync.db')

        self.conn = sqlite3.connect(dbpath)

        self.conn.execute(SQL_CREATE_FILES_TABLE)

        self.conn.commit()

    def current_hash(self, path):

        """Return the hash of the file as it exists now in the filesystem"""

        fullpath = join(self.syncdir, path)

        if fullpath in self.deleted_dirs:

            return 'DEL'

        if isdir(fullpath):

            return 'DIR'

        return merkle(fullpath)

    def delete_inactive(self, timestamp):

        sql = 'DELETE FROM files WHERE timestamp != ?'

        self.conn.execute(sql, (timestamp,))

        self.conn.commit()

    def download(self, path, hash):

        fullpath = join(self.syncdir, path)

        if hash == 'DEL':

            else:

                print 'Downloading %s...' % path

                download(client, self.container, path, fullpath)

        current = self.current_hash(path)

        assert current == hash, "Downloaded file does not match hash"

        self.save(path, hash)

    def empty_trash(self):

        for filename in os.listdir(self.trashdir):

            path = join(self.trashdir, filename)

            os.remove(path)

    def find_hash(self, hash):

        sql = 'SELECT path FROM files WHERE hash = ?'

        ret = self.conn.execute(sql, (hash,)).fetchone()

        if ret:

            return join(self.syncdir, ret[0])

        if hash in os.listdir(self.trashdir):

            return join(self.trashdir, hash)

        return None

    def previous_hash(self, path):

        """Return the hash of the file according to the previous sync with

           the server. Return DEL if not such entry exists."""

        sql = 'SELECT hash FROM files WHERE path = ?'

        ret = self.conn.execute(sql, (path,)).fetchone()

        return ret[0] if ret else 'DEL'

    def remote_hash(self, path):

        """Return the hash of the file according to the server"""

        try:

            meta = client.retrieve_object_metadata(self.container, path)

        except Fault:

            return 'DIR'

        else:

            return meta['x-object-hash']

    def remove_deleted_dirs(self):

        for path in sorted(self.deleted_dirs, key=len, reverse=True):

            os.rmdir(path)

            self.deleted_dirs.remove(path)

    def resolve_conflict(self, path, hash):

        """Resolve a sync conflict by renaming the local file and downloading

           the remote one."""

        fullpath = join(self.syncdir, path)

        resolved = fullpath + '.local'

        i = 0

        while exists(resolved):

            i += 1

            resolved = fullpath + '.local%d' % i

        os.rename(fullpath, resolved)

        self.download(path, hash)

    def rmdir(self, path):

        """Remove a dir or mark for deletion if non-empty

        If a dir is empty delete it and check if any of its parents should be

        deleted too. Else mark it for later deletion.

        """

        fullpath = join(self.syncdir, path)

        if not exists(fullpath):

            return

        if os.listdir(fullpath):

            # Directory not empty

            self.deleted_dirs.add(fullpath)

            return

        os.rmdir(fullpath)

        self.deleted_dirs.discard(fullpath)

        parent = dirname(fullpath)

        while parent in self.deleted_dirs:

            os.rmdir(parent)

            self.deleted_dirs.remove(parent)

            parent = dirname(parent)

    def save(self, path, hash):

        """Save the hash value of a file. This value will be later returned

           by `previous_hash`."""

        sql = 'INSERT OR REPLACE INTO files (path, hash) VALUES (?, ?)'

        self.conn.execute(sql, (path, hash))

        self.conn.commit()

    def touch(self, path, now):

        sql = 'UPDATE files SET timestamp = ? WHERE path = ?'

        self.conn.execute(sql, (now, path))

        self.conn.commit()

    def trash(self, path):

        """Move a file to trash or delete it if it's a directory"""

        fullpath = join(self.syncdir, path)

        if not exists(fullpath):

            return

        if isfile(fullpath):

            hash = merkle(fullpath)

            trashpath = join(self.trashdir, hash)

            os.rename(fullpath, trashpath)

        else:

            self.rmdir(path)

    def upload(self, path, hash):

        fullpath = join(self.syncdir, path)

        if hash == 'DEL':

                prefix += '/'

            print 'Uploading %s...' % path

            upload(client, fullpath, self.container, prefix, name)

        remote = self.remote_hash(path)

        assert remote == hash, "Uploaded file does not match hash"

        self.save(path, hash)

    previous = state.previous_hash(path)

    current = state.current_hash(path)

    remote = state.remote_hash(path)

    if current == previous:

        # No local changes, download any remote changes

        if remote != previous:

def walk(dir, container):

    """Iterates on the files of the hierarchy created by merging the files

       in `dir` and the objects in `container`."""

    pending = ['']

    while pending:

        dirs = set()

        files = set()

            continue

        if root:

            yield root

        dirpath = join(dir, root)

        if exists(dirpath):

            for filename in os.listdir(dirpath):

                    dirs.add(path)

                else:

                    files.add(path)

        for object in client.list_objects(container, format='json',

            if 'subdir' in object:

                continue

            name = object['name']

                dirs.add(name)

            else:

                files.add(name)

        pending += sorted(dirs)

        for path in files:

            yield path

    if len(sys.argv) != 2:

        print 'syntax: %s <dir>' % sys.argv[0]

        sys.exit(1)

    syncdir = sys.argv[1]

    _makedirs(SETTINGS_DIR)

    container = os.environ.get('PITHOS_SYNC_CONTAINER', DEFAULT_CONTAINER)

    client.create_container(container)

    state = State(syncdir, container)

    now = int(time())

    for path in walk(syncdir, container):

        print 'Syncing', path

        sync(path, state)

        state.touch(path, now)

    state.delete_inactive(now)

    state.empty_trash()

    state.remove_deleted_dirs()