root / pithos / tools / pithos-sync @ 2bdc9dc4
History | View | Annotate | Download (10.4 kB)
1 | d8c26d94 | Giorgos Verigakis | #!/usr/bin/env python |
---|---|---|---|
2 | d8c26d94 | Giorgos Verigakis | |
3 | eaa6fb55 | Antony Chazapis | # Copyright 2011 GRNET S.A. All rights reserved. |
4 | eaa6fb55 | Antony Chazapis | # |
5 | eaa6fb55 | Antony Chazapis | # Redistribution and use in source and binary forms, with or |
6 | eaa6fb55 | Antony Chazapis | # without modification, are permitted provided that the following |
7 | eaa6fb55 | Antony Chazapis | # conditions are met: |
8 | eaa6fb55 | Antony Chazapis | # |
9 | eaa6fb55 | Antony Chazapis | # 1. Redistributions of source code must retain the above |
10 | eaa6fb55 | Antony Chazapis | # copyright notice, this list of conditions and the following |
11 | eaa6fb55 | Antony Chazapis | # disclaimer. |
12 | eaa6fb55 | Antony Chazapis | # |
13 | eaa6fb55 | Antony Chazapis | # 2. Redistributions in binary form must reproduce the above |
14 | eaa6fb55 | Antony Chazapis | # copyright notice, this list of conditions and the following |
15 | eaa6fb55 | Antony Chazapis | # disclaimer in the documentation and/or other materials |
16 | eaa6fb55 | Antony Chazapis | # provided with the distribution. |
17 | eaa6fb55 | Antony Chazapis | # |
18 | eaa6fb55 | Antony Chazapis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
19 | eaa6fb55 | Antony Chazapis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
20 | eaa6fb55 | Antony Chazapis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
21 | eaa6fb55 | Antony Chazapis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
22 | eaa6fb55 | Antony Chazapis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
23 | eaa6fb55 | Antony Chazapis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
24 | eaa6fb55 | Antony Chazapis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
25 | eaa6fb55 | Antony Chazapis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
26 | eaa6fb55 | Antony Chazapis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
27 | eaa6fb55 | Antony Chazapis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
28 | eaa6fb55 | Antony Chazapis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
29 | eaa6fb55 | Antony Chazapis | # POSSIBILITY OF SUCH DAMAGE. |
30 | eaa6fb55 | Antony Chazapis | # |
31 | eaa6fb55 | Antony Chazapis | # The views and conclusions contained in the software and |
32 | eaa6fb55 | Antony Chazapis | # documentation are those of the authors and should not be |
33 | eaa6fb55 | Antony Chazapis | # interpreted as representing official policies, either expressed |
34 | eaa6fb55 | Antony Chazapis | # or implied, of GRNET S.A. |
35 | eaa6fb55 | Antony Chazapis | |
36 | d8c26d94 | Giorgos Verigakis | import os |
37 | d8c26d94 | Giorgos Verigakis | import sqlite3 |
38 | d8c26d94 | Giorgos Verigakis | import sys |
39 | d8c26d94 | Giorgos Verigakis | |
40 | c0b10ca6 | Giorgos Verigakis | from os.path import exists, expanduser, isdir, isfile, join, split |
41 | c0b10ca6 | Giorgos Verigakis | from shutil import copyfile |
42 | c0b10ca6 | Giorgos Verigakis | from time import time |
43 | c0b10ca6 | Giorgos Verigakis | |
44 | 5a96180b | Antony Chazapis | from pithos.lib.transfer import download, upload |
45 | 5a96180b | Antony Chazapis | from pithos.lib.client import Pithos_Client, Fault |
46 | 5a96180b | Antony Chazapis | from pithos.lib.hashmap import merkle |
47 | 5a96180b | Antony Chazapis | from pithos.lib.util import get_user, get_auth, get_server |
48 | 68d89033 | Giorgos Verigakis | |
49 | d8c26d94 | Giorgos Verigakis | |
50 | b0762fe3 | Antony Chazapis | DEFAULT_CONTAINER = 'pithos' |
51 | c0b10ca6 | Giorgos Verigakis | SETTINGS_DIR = expanduser('~/.pithos') |
52 | c0b10ca6 | Giorgos Verigakis | TRASH_DIR = '.pithos_trash' |
53 | b0762fe3 | Antony Chazapis | |
54 | c0b10ca6 | Giorgos Verigakis | SQL_CREATE_FILES_TABLE = '''CREATE TABLE IF NOT EXISTS files ( |
55 | c0b10ca6 | Giorgos Verigakis | path TEXT PRIMARY KEY, |
56 | c0b10ca6 | Giorgos Verigakis | hash TEXT, |
57 | c0b10ca6 | Giorgos Verigakis | timestamp INTEGER)''' |
58 | d8c26d94 | Giorgos Verigakis | |
59 | 99d0e277 | Antony Chazapis | |
60 | c0b10ca6 | Giorgos Verigakis | client = Pithos_Client(get_server(), get_auth(), get_user()) |
61 | d8c26d94 | Giorgos Verigakis | |
62 | db3c7c26 | Antony Chazapis | |
63 | c0b10ca6 | Giorgos Verigakis | def _makedirs(path): |
64 | c0b10ca6 | Giorgos Verigakis | try: |
65 | c0b10ca6 | Giorgos Verigakis | os.makedirs(path) |
66 | c0b10ca6 | Giorgos Verigakis | except OSError: |
67 | c0b10ca6 | Giorgos Verigakis | pass |
68 | db3c7c26 | Antony Chazapis | |
69 | 68d89033 | Giorgos Verigakis | |
70 | c0b10ca6 | Giorgos Verigakis | class State(object): |
71 | c0b10ca6 | Giorgos Verigakis | def __init__(self, syncdir, container): |
72 | c0b10ca6 | Giorgos Verigakis | self.syncdir = syncdir |
73 | c0b10ca6 | Giorgos Verigakis | self.container = container |
74 | c0b10ca6 | Giorgos Verigakis | self.trashdir = join(syncdir, TRASH_DIR) |
75 | c0b10ca6 | Giorgos Verigakis | self.deleted_dirs = set() |
76 | db3c7c26 | Antony Chazapis | |
77 | c0b10ca6 | Giorgos Verigakis | _makedirs(self.trashdir) |
78 | 99d0e277 | Antony Chazapis | |
79 | c0b10ca6 | Giorgos Verigakis | dbpath = join(SETTINGS_DIR, 'sync.db') |
80 | 99d0e277 | Antony Chazapis | self.conn = sqlite3.connect(dbpath) |
81 | c0b10ca6 | Giorgos Verigakis | self.conn.execute(SQL_CREATE_FILES_TABLE) |
82 | 99d0e277 | Antony Chazapis | self.conn.commit() |
83 | 99d0e277 | Antony Chazapis | |
84 | c0b10ca6 | Giorgos Verigakis | def current_hash(self, path): |
85 | c0b10ca6 | Giorgos Verigakis | """Return the hash of the file as it exists now in the filesystem""" |
86 | c0b10ca6 | Giorgos Verigakis | |
87 | c0b10ca6 | Giorgos Verigakis | fullpath = join(self.syncdir, path) |
88 | c0b10ca6 | Giorgos Verigakis | if fullpath in self.deleted_dirs: |
89 | c0b10ca6 | Giorgos Verigakis | return 'DEL' |
90 | c0b10ca6 | Giorgos Verigakis | if not exists(fullpath): |
91 | c0b10ca6 | Giorgos Verigakis | return 'DEL' |
92 | c0b10ca6 | Giorgos Verigakis | if isdir(fullpath): |
93 | c0b10ca6 | Giorgos Verigakis | return 'DIR' |
94 | c0b10ca6 | Giorgos Verigakis | return merkle(fullpath) |
95 | 99d0e277 | Antony Chazapis | |
96 | c0b10ca6 | Giorgos Verigakis | def delete_inactive(self, timestamp): |
97 | c0b10ca6 | Giorgos Verigakis | sql = 'DELETE FROM files WHERE timestamp != ?' |
98 | c0b10ca6 | Giorgos Verigakis | self.conn.execute(sql, (timestamp,)) |
99 | 99d0e277 | Antony Chazapis | self.conn.commit() |
100 | 99d0e277 | Antony Chazapis | |
101 | c0b10ca6 | Giorgos Verigakis | def download(self, path, hash): |
102 | c0b10ca6 | Giorgos Verigakis | fullpath = join(self.syncdir, path) |
103 | c0b10ca6 | Giorgos Verigakis | if hash == 'DEL': |
104 | c0b10ca6 | Giorgos Verigakis | self.trash(path) |
105 | c0b10ca6 | Giorgos Verigakis | elif hash == 'DIR': |
106 | c0b10ca6 | Giorgos Verigakis | _makedirs(fullpath) |
107 | c0b10ca6 | Giorgos Verigakis | else: |
108 | c0b10ca6 | Giorgos Verigakis | self.trash(path) # Trash any old version |
109 | c0b10ca6 | Giorgos Verigakis | localpath = self.find_hash(hash) |
110 | c0b10ca6 | Giorgos Verigakis | if localpath: |
111 | c0b10ca6 | Giorgos Verigakis | copyfile(localpath, fullpath) |
112 | c0b10ca6 | Giorgos Verigakis | else: |
113 | c0b10ca6 | Giorgos Verigakis | print 'Downloading %s...' % path |
114 | c0b10ca6 | Giorgos Verigakis | download(client, self.container, path, fullpath) |
115 | db3c7c26 | Antony Chazapis | |
116 | c0b10ca6 | Giorgos Verigakis | current = self.current_hash(path) |
117 | c0b10ca6 | Giorgos Verigakis | assert current == hash, "Downloaded file does not match hash" |
118 | c0b10ca6 | Giorgos Verigakis | self.save(path, hash) |
119 | d8c26d94 | Giorgos Verigakis | |
120 | c0b10ca6 | Giorgos Verigakis | def empty_trash(self): |
121 | c0b10ca6 | Giorgos Verigakis | for filename in os.listdir(self.trashdir): |
122 | c0b10ca6 | Giorgos Verigakis | path = join(self.trashdir, filename) |
123 | c0b10ca6 | Giorgos Verigakis | os.remove(path) |
124 | d8c26d94 | Giorgos Verigakis | |
125 | c0b10ca6 | Giorgos Verigakis | def find_hash(self, hash): |
126 | 99d0e277 | Antony Chazapis | sql = 'SELECT path FROM files WHERE hash = ?' |
127 | 99d0e277 | Antony Chazapis | ret = self.conn.execute(sql, (hash,)).fetchone() |
128 | c0b10ca6 | Giorgos Verigakis | if ret: |
129 | c0b10ca6 | Giorgos Verigakis | return join(self.syncdir, ret[0]) |
130 | c0b10ca6 | Giorgos Verigakis | |
131 | c0b10ca6 | Giorgos Verigakis | if hash in os.listdir(self.trashdir): |
132 | c0b10ca6 | Giorgos Verigakis | return join(self.trashdir, hash) |
133 | c0b10ca6 | Giorgos Verigakis | |
134 | c0b10ca6 | Giorgos Verigakis | return None |
135 | d8c26d94 | Giorgos Verigakis | |
136 | c0b10ca6 | Giorgos Verigakis | def previous_hash(self, path): |
137 | c0b10ca6 | Giorgos Verigakis | """Return the hash of the file according to the previous sync with |
138 | c0b10ca6 | Giorgos Verigakis | the server. Return DEL if not such entry exists.""" |
139 | c0b10ca6 | Giorgos Verigakis | |
140 | c0b10ca6 | Giorgos Verigakis | sql = 'SELECT hash FROM files WHERE path = ?' |
141 | c0b10ca6 | Giorgos Verigakis | ret = self.conn.execute(sql, (path,)).fetchone() |
142 | c0b10ca6 | Giorgos Verigakis | return ret[0] if ret else 'DEL' |
143 | 68d89033 | Giorgos Verigakis | |
144 | c0b10ca6 | Giorgos Verigakis | def remote_hash(self, path): |
145 | c0b10ca6 | Giorgos Verigakis | """Return the hash of the file according to the server""" |
146 | c0b10ca6 | Giorgos Verigakis | |
147 | d8c26d94 | Giorgos Verigakis | try: |
148 | c0b10ca6 | Giorgos Verigakis | meta = client.retrieve_object_metadata(self.container, path) |
149 | d8c26d94 | Giorgos Verigakis | except Fault: |
150 | 68d89033 | Giorgos Verigakis | return 'DEL' |
151 | 68d89033 | Giorgos Verigakis | if meta.get('content-type', None) == 'application/directory': |
152 | 68d89033 | Giorgos Verigakis | return 'DIR' |
153 | d8c26d94 | Giorgos Verigakis | else: |
154 | 3ea35aa0 | Antony Chazapis | return meta['x-object-hash'] |
155 | c0b10ca6 | Giorgos Verigakis | |
156 | c0b10ca6 | Giorgos Verigakis | def remove_deleted_dirs(self): |
157 | c0b10ca6 | Giorgos Verigakis | for path in sorted(self.deleted_dirs, key=len, reverse=True): |
158 | c0b10ca6 | Giorgos Verigakis | os.rmdir(path) |
159 | c0b10ca6 | Giorgos Verigakis | self.deleted_dirs.remove(path) |
160 | c0b10ca6 | Giorgos Verigakis | |
161 | c0b10ca6 | Giorgos Verigakis | def resolve_conflict(self, path, hash): |
162 | c0b10ca6 | Giorgos Verigakis | """Resolve a sync conflict by renaming the local file and downloading |
163 | c0b10ca6 | Giorgos Verigakis | the remote one.""" |
164 | c0b10ca6 | Giorgos Verigakis | |
165 | c0b10ca6 | Giorgos Verigakis | fullpath = join(self.syncdir, path) |
166 | c0b10ca6 | Giorgos Verigakis | resolved = fullpath + '.local' |
167 | c0b10ca6 | Giorgos Verigakis | i = 0 |
168 | c0b10ca6 | Giorgos Verigakis | while exists(resolved): |
169 | c0b10ca6 | Giorgos Verigakis | i += 1 |
170 | c0b10ca6 | Giorgos Verigakis | resolved = fullpath + '.local%d' % i |
171 | c0b10ca6 | Giorgos Verigakis | |
172 | c0b10ca6 | Giorgos Verigakis | os.rename(fullpath, resolved) |
173 | c0b10ca6 | Giorgos Verigakis | self.download(path, hash) |
174 | c0b10ca6 | Giorgos Verigakis | |
175 | c0b10ca6 | Giorgos Verigakis | def rmdir(self, path): |
176 | c0b10ca6 | Giorgos Verigakis | """Remove a dir or mark for deletion if non-empty |
177 | c0b10ca6 | Giorgos Verigakis | |
178 | c0b10ca6 | Giorgos Verigakis | If a dir is empty delete it and check if any of its parents should be |
179 | c0b10ca6 | Giorgos Verigakis | deleted too. Else mark it for later deletion. |
180 | c0b10ca6 | Giorgos Verigakis | """ |
181 | c0b10ca6 | Giorgos Verigakis | |
182 | c0b10ca6 | Giorgos Verigakis | fullpath = join(self.syncdir, path) |
183 | c0b10ca6 | Giorgos Verigakis | if not exists(fullpath): |
184 | c0b10ca6 | Giorgos Verigakis | return |
185 | c0b10ca6 | Giorgos Verigakis | |
186 | c0b10ca6 | Giorgos Verigakis | if os.listdir(fullpath): |
187 | c0b10ca6 | Giorgos Verigakis | # Directory not empty |
188 | c0b10ca6 | Giorgos Verigakis | self.deleted_dirs.add(fullpath) |
189 | c0b10ca6 | Giorgos Verigakis | return |
190 | c0b10ca6 | Giorgos Verigakis | |
191 | c0b10ca6 | Giorgos Verigakis | os.rmdir(fullpath) |
192 | c0b10ca6 | Giorgos Verigakis | self.deleted_dirs.discard(fullpath) |
193 | c0b10ca6 | Giorgos Verigakis | |
194 | c0b10ca6 | Giorgos Verigakis | parent = dirname(fullpath) |
195 | c0b10ca6 | Giorgos Verigakis | while parent in self.deleted_dirs: |
196 | c0b10ca6 | Giorgos Verigakis | os.rmdir(parent) |
197 | c0b10ca6 | Giorgos Verigakis | self.deleted_dirs.remove(parent) |
198 | c0b10ca6 | Giorgos Verigakis | parent = dirname(parent) |
199 | c0b10ca6 | Giorgos Verigakis | |
200 | c0b10ca6 | Giorgos Verigakis | def save(self, path, hash): |
201 | c0b10ca6 | Giorgos Verigakis | """Save the hash value of a file. This value will be later returned |
202 | c0b10ca6 | Giorgos Verigakis | by `previous_hash`.""" |
203 | c0b10ca6 | Giorgos Verigakis | |
204 | c0b10ca6 | Giorgos Verigakis | sql = 'INSERT OR REPLACE INTO files (path, hash) VALUES (?, ?)' |
205 | c0b10ca6 | Giorgos Verigakis | self.conn.execute(sql, (path, hash)) |
206 | c0b10ca6 | Giorgos Verigakis | self.conn.commit() |
207 | c0b10ca6 | Giorgos Verigakis | |
208 | c0b10ca6 | Giorgos Verigakis | def touch(self, path, now): |
209 | c0b10ca6 | Giorgos Verigakis | sql = 'UPDATE files SET timestamp = ? WHERE path = ?' |
210 | c0b10ca6 | Giorgos Verigakis | self.conn.execute(sql, (now, path)) |
211 | c0b10ca6 | Giorgos Verigakis | self.conn.commit() |
212 | c0b10ca6 | Giorgos Verigakis | |
213 | c0b10ca6 | Giorgos Verigakis | def trash(self, path): |
214 | c0b10ca6 | Giorgos Verigakis | """Move a file to trash or delete it if it's a directory""" |
215 | c0b10ca6 | Giorgos Verigakis | |
216 | c0b10ca6 | Giorgos Verigakis | fullpath = join(self.syncdir, path) |
217 | c0b10ca6 | Giorgos Verigakis | if not exists(fullpath): |
218 | c0b10ca6 | Giorgos Verigakis | return |
219 | c0b10ca6 | Giorgos Verigakis | |
220 | c0b10ca6 | Giorgos Verigakis | if isfile(fullpath): |
221 | c0b10ca6 | Giorgos Verigakis | hash = merkle(fullpath) |
222 | c0b10ca6 | Giorgos Verigakis | trashpath = join(self.trashdir, hash) |
223 | c0b10ca6 | Giorgos Verigakis | os.rename(fullpath, trashpath) |
224 | db3c7c26 | Antony Chazapis | else: |
225 | c0b10ca6 | Giorgos Verigakis | self.rmdir(path) |
226 | c0b10ca6 | Giorgos Verigakis | |
227 | c0b10ca6 | Giorgos Verigakis | def upload(self, path, hash): |
228 | c0b10ca6 | Giorgos Verigakis | fullpath = join(self.syncdir, path) |
229 | c0b10ca6 | Giorgos Verigakis | if hash == 'DEL': |
230 | c0b10ca6 | Giorgos Verigakis | client.delete_object(self.container, path) |
231 | c0b10ca6 | Giorgos Verigakis | elif hash == 'DIR': |
232 | c0b10ca6 | Giorgos Verigakis | client.create_directory_marker(self.container, path) |
233 | c0b10ca6 | Giorgos Verigakis | else: |
234 | c0b10ca6 | Giorgos Verigakis | prefix, name = split(path) |
235 | c0b10ca6 | Giorgos Verigakis | if prefix: |
236 | c0b10ca6 | Giorgos Verigakis | prefix += '/' |
237 | c0b10ca6 | Giorgos Verigakis | print 'Uploading %s...' % path |
238 | c0b10ca6 | Giorgos Verigakis | upload(client, fullpath, self.container, prefix, name) |
239 | c0b10ca6 | Giorgos Verigakis | |
240 | c0b10ca6 | Giorgos Verigakis | remote = self.remote_hash(path) |
241 | c0b10ca6 | Giorgos Verigakis | assert remote == hash, "Uploaded file does not match hash" |
242 | c0b10ca6 | Giorgos Verigakis | self.save(path, hash) |
243 | 68d89033 | Giorgos Verigakis | |
244 | 68d89033 | Giorgos Verigakis | |
245 | c0b10ca6 | Giorgos Verigakis | def sync(path, state): |
246 | c0b10ca6 | Giorgos Verigakis | previous = state.previous_hash(path) |
247 | c0b10ca6 | Giorgos Verigakis | current = state.current_hash(path) |
248 | c0b10ca6 | Giorgos Verigakis | remote = state.remote_hash(path) |
249 | d8c26d94 | Giorgos Verigakis | |
250 | c0b10ca6 | Giorgos Verigakis | if current == previous: |
251 | c0b10ca6 | Giorgos Verigakis | # No local changes, download any remote changes |
252 | c0b10ca6 | Giorgos Verigakis | if remote != previous: |
253 | c0b10ca6 | Giorgos Verigakis | state.download(path, remote) |
254 | c0b10ca6 | Giorgos Verigakis | elif remote == previous: |
255 | c0b10ca6 | Giorgos Verigakis | # No remote changes, upload any local changes |
256 | c0b10ca6 | Giorgos Verigakis | if current != previous: |
257 | c0b10ca6 | Giorgos Verigakis | state.upload(path, current) |
258 | d8c26d94 | Giorgos Verigakis | else: |
259 | c0b10ca6 | Giorgos Verigakis | # Both local and remote file have changes since last sync |
260 | c0b10ca6 | Giorgos Verigakis | if current == remote: |
261 | c0b10ca6 | Giorgos Verigakis | state.save(path, remote) # Local and remote changes match |
262 | c0b10ca6 | Giorgos Verigakis | else: |
263 | c0b10ca6 | Giorgos Verigakis | state.resolve_conflict(path, remote) |
264 | 68d89033 | Giorgos Verigakis | |
265 | 68d89033 | Giorgos Verigakis | |
266 | c0b10ca6 | Giorgos Verigakis | def walk(dir, container): |
267 | c0b10ca6 | Giorgos Verigakis | """Iterates on the files of the hierarchy created by merging the files |
268 | c0b10ca6 | Giorgos Verigakis | in `dir` and the objects in `container`.""" |
269 | c0b10ca6 | Giorgos Verigakis | |
270 | 68d89033 | Giorgos Verigakis | pending = [''] |
271 | 68d89033 | Giorgos Verigakis | |
272 | 68d89033 | Giorgos Verigakis | while pending: |
273 | 68d89033 | Giorgos Verigakis | dirs = set() |
274 | 68d89033 | Giorgos Verigakis | files = set() |
275 | c0b10ca6 | Giorgos Verigakis | root = pending.pop(0) # Depth First Traversal |
276 | c0b10ca6 | Giorgos Verigakis | if root == TRASH_DIR: |
277 | c0b10ca6 | Giorgos Verigakis | continue |
278 | 68d89033 | Giorgos Verigakis | if root: |
279 | 68d89033 | Giorgos Verigakis | yield root |
280 | 68d89033 | Giorgos Verigakis | |
281 | c0b10ca6 | Giorgos Verigakis | dirpath = join(dir, root) |
282 | c0b10ca6 | Giorgos Verigakis | if exists(dirpath): |
283 | 68d89033 | Giorgos Verigakis | for filename in os.listdir(dirpath): |
284 | c0b10ca6 | Giorgos Verigakis | path = join(root, filename) |
285 | c0b10ca6 | Giorgos Verigakis | if isdir(join(dir, path)): |
286 | 68d89033 | Giorgos Verigakis | dirs.add(path) |
287 | 68d89033 | Giorgos Verigakis | else: |
288 | 68d89033 | Giorgos Verigakis | files.add(path) |
289 | 68d89033 | Giorgos Verigakis | |
290 | c0b10ca6 | Giorgos Verigakis | for object in client.list_objects(container, format='json', |
291 | c0b10ca6 | Giorgos Verigakis | prefix=root, delimiter='/'): |
292 | 68d89033 | Giorgos Verigakis | if 'subdir' in object: |
293 | 68d89033 | Giorgos Verigakis | continue |
294 | c0b10ca6 | Giorgos Verigakis | name = object['name'] |
295 | 68d89033 | Giorgos Verigakis | if object['content_type'] == 'application/directory': |
296 | 68d89033 | Giorgos Verigakis | dirs.add(name) |
297 | 68d89033 | Giorgos Verigakis | else: |
298 | 68d89033 | Giorgos Verigakis | files.add(name) |
299 | 68d89033 | Giorgos Verigakis | |
300 | 68d89033 | Giorgos Verigakis | pending += sorted(dirs) |
301 | 68d89033 | Giorgos Verigakis | for path in files: |
302 | 68d89033 | Giorgos Verigakis | yield path |
303 | d8c26d94 | Giorgos Verigakis | |
304 | d8c26d94 | Giorgos Verigakis | |
305 | d8c26d94 | Giorgos Verigakis | def main(): |
306 | d8c26d94 | Giorgos Verigakis | if len(sys.argv) != 2: |
307 | d8c26d94 | Giorgos Verigakis | print 'syntax: %s <dir>' % sys.argv[0] |
308 | d8c26d94 | Giorgos Verigakis | sys.exit(1) |
309 | d8c26d94 | Giorgos Verigakis | |
310 | c0b10ca6 | Giorgos Verigakis | syncdir = sys.argv[1] |
311 | 68d89033 | Giorgos Verigakis | |
312 | c0b10ca6 | Giorgos Verigakis | _makedirs(SETTINGS_DIR) |
313 | c0b10ca6 | Giorgos Verigakis | container = os.environ.get('PITHOS_SYNC_CONTAINER', DEFAULT_CONTAINER) |
314 | c0b10ca6 | Giorgos Verigakis | client.create_container(container) |
315 | db3c7c26 | Antony Chazapis | |
316 | c0b10ca6 | Giorgos Verigakis | state = State(syncdir, container) |
317 | 99d0e277 | Antony Chazapis | |
318 | c0b10ca6 | Giorgos Verigakis | now = int(time()) |
319 | c0b10ca6 | Giorgos Verigakis | for path in walk(syncdir, container): |
320 | 68d89033 | Giorgos Verigakis | print 'Syncing', path |
321 | c0b10ca6 | Giorgos Verigakis | sync(path, state) |
322 | c0b10ca6 | Giorgos Verigakis | state.touch(path, now) |
323 | db3c7c26 | Antony Chazapis | |
324 | c0b10ca6 | Giorgos Verigakis | state.delete_inactive(now) |
325 | c0b10ca6 | Giorgos Verigakis | state.empty_trash() |
326 | c0b10ca6 | Giorgos Verigakis | state.remove_deleted_dirs() |
327 | d8c26d94 | Giorgos Verigakis | |
328 | d8c26d94 | Giorgos Verigakis | |
329 | d8c26d94 | Giorgos Verigakis | if __name__ == '__main__': |
330 | d8c26d94 | Giorgos Verigakis | main() |