3 # Copyright 2011 GRNET S.A. All rights reserved.
5 # Redistribution and use in source and binary forms, with or
6 # without modification, are permitted provided that the following
9 # 1. Redistributions of source code must retain the above
10 # copyright notice, this list of conditions and the following
13 # 2. Redistributions in binary form must reproduce the above
14 # copyright notice, this list of conditions and the following
15 # disclaimer in the documentation and/or other materials
16 # provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 # POSSIBILITY OF SUCH DAMAGE.
31 # The views and conclusions contained in the software and
32 # documentation are those of the authors and should not be
33 # interpreted as representing official policies, either expressed
34 # or implied, of GRNET S.A.
42 from lib import transfer
43 from lib.client import Pithos_Client, Fault
44 from lib.hashmap import HashMap, merkle
45 from lib.util import get_user, get_auth, get_server
48 DEFAULT_CONTAINER = 'pithos'
52 return os.environ['PITHOS_SYNC_CONTAINER']
54 return DEFAULT_CONTAINER
58 if not os.path.exists(path):
60 if not os.path.isdir(path):
61 raise RuntimeError("Cannot open '%s'" % (path,))
64 def copy_file(src, dst):
65 print '***', 'COPYING', src, dst
66 path = os.path.dirname(dst)
68 shutil.copyfile(src, dst)
82 self.path = os.path.join(confdir, 'trash')
85 dbpath = os.path.join(confdir, 'trash.db')
86 self.conn = sqlite3.connect(dbpath)
87 sql = '''CREATE TABLE IF NOT EXISTS files (
88 path TEXT PRIMARY KEY, hash TEXT)'''
89 self.conn.execute(sql)
92 def put(self, fullpath, path, hash):
93 copy_file(fullpath, os.path.join(self.path, path))
95 sql = 'INSERT OR REPLACE INTO files VALUES (?, ?)'
96 self.conn.execute(sql, (path, hash))
99 def search(self, hash):
100 sql = 'SELECT path FROM files WHERE hash = ?'
101 ret = self.conn.execute(sql, (hash,)).fetchone()
102 return ret[0] if ret else None
105 sql = 'DELETE FROM files'
106 self.conn.execute(sql)
108 shutil.rmtree(self.path)
110 def fullpath(self, path):
111 return os.path.join(self.path, path)
114 class LocalState(object):
115 def __init__(self, path):
118 dbpath = os.path.join(confdir, 'state.db')
119 self.conn = sqlite3.connect(dbpath)
120 sql = '''CREATE TABLE IF NOT EXISTS files (
121 path TEXT PRIMARY KEY, hash TEXT)'''
122 self.conn.execute(sql)
126 sql = 'SELECT hash FROM files WHERE path = ?'
127 ret = self.conn.execute(sql, (path,)).fetchone()
128 return ret[0] if ret else 'DEL'
130 def put(self, path, hash):
131 sql = 'INSERT OR REPLACE INTO files VALUES (?, ?)'
132 self.conn.execute(sql, (path, hash))
135 def search(self, hash):
136 sql = 'SELECT path FROM files WHERE hash = ?'
137 ret = self.conn.execute(sql, (hash,)).fetchone()
138 return ret[0] if ret else None
140 def fullpath(self, path):
141 return os.path.join(self.path, path)
144 class CurrentState(object):
145 def __init__(self, path):
149 fullpath = os.path.join(self.path, path)
150 if os.path.exists(fullpath):
151 if os.path.isdir(fullpath):
154 return merkle(fullpath, conf['blocksize'], conf['blockhash'])
158 def fullpath(self, path):
159 return os.path.join(self.path, path)
162 class RemoteState(object):
163 def __init__(self, client):
165 self.container = get_container()
169 meta = self.client.retrieve_object_metadata(self.container, path)
172 if meta.get('content-type', None) == 'application/directory':
175 return meta['x-object-hash']
178 def update_local(path, S):
179 # XXX If something is already here, put it in trash and delete it.
180 # XXX If we have a directory already here, put all files in trash.
181 fullpath = cstate.fullpath(path)
183 trash.put(fullpath, path, S)
185 if os.path.exists(fullpath):
186 trash.put(fullpath, path, S)
187 # XXX Strip trailing slash (or escape).
190 # First, search for local copy
191 file = lstate.search(S)
193 copy_file(lstate.fullpath(file), fullpath)
195 # Search for copy in trash
196 file = trash.search(S)
198 # XXX Move from trash (not copy).
199 copy_file(trash.fullpath(file), fullpath)
202 transfer.download(client, get_container(), path, fullpath)
203 assert cstate.get(path) == S
206 def update_remote(path, S):
207 fullpath = cstate.fullpath(path)
209 client.delete_object(get_container(), path)
211 client.create_directory_marker(get_container(), path)
213 prefix, name = os.path.split(path)
216 transfer.upload(client, fullpath, get_container(), prefix, name)
217 assert rstate.get(path) == S
220 def resolve_conflict(path):
221 # XXX Check if this works with dirs.
222 fullpath = cstate.fullpath(path)
223 if os.path.exists(fullpath):
224 os.rename(fullpath, fullpath + '.local')
235 update_local(path, R)
242 update_remote(path, C)
246 # At this point both local and remote states have changes since last sync
249 # We were lucky, both had the same change
252 # Conflict, try to resolve it
253 resolve_conflict(path)
254 update_local(path, R)
264 root = pending.pop(0)
268 dirpath = os.path.join(dir, root)
269 if os.path.exists(dirpath):
270 for filename in os.listdir(dirpath):
271 path = os.path.join(root, filename)
272 if os.path.isdir(os.path.join(dir, path)):
277 for object in client.list_objects(get_container(), prefix=root,
278 delimiter='/', format='json'):
280 if 'subdir' in object:
282 name = str(object['name'])
283 if object['content_type'] == 'application/directory':
288 pending += sorted(dirs)
294 global client, conf, confdir, trash, lstate, cstate, rstate
296 if len(sys.argv) != 2:
297 print 'syntax: %s <dir>' % sys.argv[0]
301 client = Pithos_Client(get_server(), get_auth(), get_user())
303 container = get_container()
305 meta = client.retrieve_container_metadata(container)
307 raise RuntimeError("Cannot open container '%s'" % (container,))
309 conf = {'local': dir,
311 'blocksize': int(meta['x-container-block-size']),
312 'blockhash': meta['x-container-block-hash']}
313 confdir = os.path.expanduser('~/.pithos-sync/')
315 conffile = os.path.join(confdir, 'config')
316 if os.path.isfile(conffile):
318 if (conf != pickle.loads(open(conffile, 'rb').read())):
321 shutil.rmtree(confdir)
325 lstate = LocalState(dir)
326 cstate = CurrentState(dir)
327 rstate = RemoteState(client)
329 for path in walk(dir):
330 print 'Syncing', path
333 f = open(conffile, 'wb')
334 f.write(pickle.dumps(conf))
340 if __name__ == '__main__':