root / snf-pithos-tools / pithos / tools / sync.py @ 9e826a59
History | View | Annotate | Download (10.4 kB)
1 | d8c26d94 | Giorgos Verigakis | #!/usr/bin/env python
|
---|---|---|---|
2 | d8c26d94 | Giorgos Verigakis | |
3 | eaa6fb55 | Antony Chazapis | # Copyright 2011 GRNET S.A. All rights reserved.
|
4 | eaa6fb55 | Antony Chazapis | #
|
5 | eaa6fb55 | Antony Chazapis | # Redistribution and use in source and binary forms, with or
|
6 | eaa6fb55 | Antony Chazapis | # without modification, are permitted provided that the following
|
7 | eaa6fb55 | Antony Chazapis | # conditions are met:
|
8 | eaa6fb55 | Antony Chazapis | #
|
9 | eaa6fb55 | Antony Chazapis | # 1. Redistributions of source code must retain the above
|
10 | eaa6fb55 | Antony Chazapis | # copyright notice, this list of conditions and the following
|
11 | eaa6fb55 | Antony Chazapis | # disclaimer.
|
12 | eaa6fb55 | Antony Chazapis | #
|
13 | eaa6fb55 | Antony Chazapis | # 2. Redistributions in binary form must reproduce the above
|
14 | eaa6fb55 | Antony Chazapis | # copyright notice, this list of conditions and the following
|
15 | eaa6fb55 | Antony Chazapis | # disclaimer in the documentation and/or other materials
|
16 | eaa6fb55 | Antony Chazapis | # provided with the distribution.
|
17 | eaa6fb55 | Antony Chazapis | #
|
18 | eaa6fb55 | Antony Chazapis | # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
19 | eaa6fb55 | Antony Chazapis | # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
20 | eaa6fb55 | Antony Chazapis | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
21 | eaa6fb55 | Antony Chazapis | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
22 | eaa6fb55 | Antony Chazapis | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23 | eaa6fb55 | Antony Chazapis | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24 | eaa6fb55 | Antony Chazapis | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
25 | eaa6fb55 | Antony Chazapis | # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
26 | eaa6fb55 | Antony Chazapis | # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
27 | eaa6fb55 | Antony Chazapis | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
28 | eaa6fb55 | Antony Chazapis | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
29 | eaa6fb55 | Antony Chazapis | # POSSIBILITY OF SUCH DAMAGE.
|
30 | eaa6fb55 | Antony Chazapis | #
|
31 | eaa6fb55 | Antony Chazapis | # The views and conclusions contained in the software and
|
32 | eaa6fb55 | Antony Chazapis | # documentation are those of the authors and should not be
|
33 | eaa6fb55 | Antony Chazapis | # interpreted as representing official policies, either expressed
|
34 | eaa6fb55 | Antony Chazapis | # or implied, of GRNET S.A.
|
35 | eaa6fb55 | Antony Chazapis | |
36 | d8c26d94 | Giorgos Verigakis | import os |
37 | d8c26d94 | Giorgos Verigakis | import sqlite3 |
38 | d8c26d94 | Giorgos Verigakis | import sys |
39 | d8c26d94 | Giorgos Verigakis | |
40 | c0b10ca6 | Giorgos Verigakis | from os.path import exists, expanduser, isdir, isfile, join, split |
41 | c0b10ca6 | Giorgos Verigakis | from shutil import copyfile |
42 | c0b10ca6 | Giorgos Verigakis | from time import time |
43 | c0b10ca6 | Giorgos Verigakis | |
44 | 5a96180b | Antony Chazapis | from pithos.lib.transfer import download, upload |
45 | 5a96180b | Antony Chazapis | from pithos.lib.client import Pithos_Client, Fault |
46 | 5a96180b | Antony Chazapis | from pithos.lib.hashmap import merkle |
47 | 5a96180b | Antony Chazapis | from pithos.lib.util import get_user, get_auth, get_server |
48 | 68d89033 | Giorgos Verigakis | |
49 | d8c26d94 | Giorgos Verigakis | |
50 | b0762fe3 | Antony Chazapis | DEFAULT_CONTAINER = 'pithos'
|
51 | c0b10ca6 | Giorgos Verigakis | SETTINGS_DIR = expanduser('~/.pithos')
|
52 | c0b10ca6 | Giorgos Verigakis | TRASH_DIR = '.pithos_trash'
|
53 | b0762fe3 | Antony Chazapis | |
54 | c0b10ca6 | Giorgos Verigakis | SQL_CREATE_FILES_TABLE = '''CREATE TABLE IF NOT EXISTS files (
|
55 | c0b10ca6 | Giorgos Verigakis | path TEXT PRIMARY KEY,
|
56 | c0b10ca6 | Giorgos Verigakis | hash TEXT,
|
57 | c0b10ca6 | Giorgos Verigakis | timestamp INTEGER)'''
|
58 | d8c26d94 | Giorgos Verigakis | |
59 | 99d0e277 | Antony Chazapis | |
60 | c0b10ca6 | Giorgos Verigakis | client = Pithos_Client(get_server(), get_auth(), get_user()) |
61 | d8c26d94 | Giorgos Verigakis | |
62 | db3c7c26 | Antony Chazapis | |
63 | c0b10ca6 | Giorgos Verigakis | def _makedirs(path): |
64 | c0b10ca6 | Giorgos Verigakis | try:
|
65 | c0b10ca6 | Giorgos Verigakis | os.makedirs(path) |
66 | c0b10ca6 | Giorgos Verigakis | except OSError: |
67 | c0b10ca6 | Giorgos Verigakis | pass
|
68 | db3c7c26 | Antony Chazapis | |
69 | 68d89033 | Giorgos Verigakis | |
70 | c0b10ca6 | Giorgos Verigakis | class State(object): |
71 | c0b10ca6 | Giorgos Verigakis | def __init__(self, syncdir, container): |
72 | c0b10ca6 | Giorgos Verigakis | self.syncdir = syncdir
|
73 | c0b10ca6 | Giorgos Verigakis | self.container = container
|
74 | c0b10ca6 | Giorgos Verigakis | self.trashdir = join(syncdir, TRASH_DIR)
|
75 | c0b10ca6 | Giorgos Verigakis | self.deleted_dirs = set() |
76 | db3c7c26 | Antony Chazapis | |
77 | c0b10ca6 | Giorgos Verigakis | _makedirs(self.trashdir)
|
78 | 99d0e277 | Antony Chazapis | |
79 | c0b10ca6 | Giorgos Verigakis | dbpath = join(SETTINGS_DIR, 'sync.db')
|
80 | 99d0e277 | Antony Chazapis | self.conn = sqlite3.connect(dbpath)
|
81 | c0b10ca6 | Giorgos Verigakis | self.conn.execute(SQL_CREATE_FILES_TABLE)
|
82 | 99d0e277 | Antony Chazapis | self.conn.commit()
|
83 | 99d0e277 | Antony Chazapis | |
84 | c0b10ca6 | Giorgos Verigakis | def current_hash(self, path): |
85 | c0b10ca6 | Giorgos Verigakis | """Return the hash of the file as it exists now in the filesystem"""
|
86 | c0b10ca6 | Giorgos Verigakis | |
87 | c0b10ca6 | Giorgos Verigakis | fullpath = join(self.syncdir, path)
|
88 | c0b10ca6 | Giorgos Verigakis | if fullpath in self.deleted_dirs: |
89 | c0b10ca6 | Giorgos Verigakis | return 'DEL' |
90 | c0b10ca6 | Giorgos Verigakis | if not exists(fullpath): |
91 | c0b10ca6 | Giorgos Verigakis | return 'DEL' |
92 | c0b10ca6 | Giorgos Verigakis | if isdir(fullpath):
|
93 | c0b10ca6 | Giorgos Verigakis | return 'DIR' |
94 | c0b10ca6 | Giorgos Verigakis | return merkle(fullpath)
|
95 | 99d0e277 | Antony Chazapis | |
96 | c0b10ca6 | Giorgos Verigakis | def delete_inactive(self, timestamp): |
97 | c0b10ca6 | Giorgos Verigakis | sql = 'DELETE FROM files WHERE timestamp != ?'
|
98 | c0b10ca6 | Giorgos Verigakis | self.conn.execute(sql, (timestamp,))
|
99 | 99d0e277 | Antony Chazapis | self.conn.commit()
|
100 | 99d0e277 | Antony Chazapis | |
101 | c0b10ca6 | Giorgos Verigakis | def download(self, path, hash): |
102 | c0b10ca6 | Giorgos Verigakis | fullpath = join(self.syncdir, path)
|
103 | c0b10ca6 | Giorgos Verigakis | if hash == 'DEL': |
104 | c0b10ca6 | Giorgos Verigakis | self.trash(path)
|
105 | c0b10ca6 | Giorgos Verigakis | elif hash == 'DIR': |
106 | c0b10ca6 | Giorgos Verigakis | _makedirs(fullpath) |
107 | c0b10ca6 | Giorgos Verigakis | else:
|
108 | c0b10ca6 | Giorgos Verigakis | self.trash(path) # Trash any old version |
109 | c0b10ca6 | Giorgos Verigakis | localpath = self.find_hash(hash) |
110 | c0b10ca6 | Giorgos Verigakis | if localpath:
|
111 | c0b10ca6 | Giorgos Verigakis | copyfile(localpath, fullpath) |
112 | c0b10ca6 | Giorgos Verigakis | else:
|
113 | c0b10ca6 | Giorgos Verigakis | print 'Downloading %s...' % path |
114 | c0b10ca6 | Giorgos Verigakis | download(client, self.container, path, fullpath)
|
115 | db3c7c26 | Antony Chazapis | |
116 | c0b10ca6 | Giorgos Verigakis | current = self.current_hash(path)
|
117 | c0b10ca6 | Giorgos Verigakis | assert current == hash, "Downloaded file does not match hash" |
118 | c0b10ca6 | Giorgos Verigakis | self.save(path, hash) |
119 | d8c26d94 | Giorgos Verigakis | |
120 | c0b10ca6 | Giorgos Verigakis | def empty_trash(self): |
121 | c0b10ca6 | Giorgos Verigakis | for filename in os.listdir(self.trashdir): |
122 | c0b10ca6 | Giorgos Verigakis | path = join(self.trashdir, filename)
|
123 | c0b10ca6 | Giorgos Verigakis | os.remove(path) |
124 | d8c26d94 | Giorgos Verigakis | |
125 | c0b10ca6 | Giorgos Verigakis | def find_hash(self, hash): |
126 | 99d0e277 | Antony Chazapis | sql = 'SELECT path FROM files WHERE hash = ?'
|
127 | 99d0e277 | Antony Chazapis | ret = self.conn.execute(sql, (hash,)).fetchone() |
128 | c0b10ca6 | Giorgos Verigakis | if ret:
|
129 | c0b10ca6 | Giorgos Verigakis | return join(self.syncdir, ret[0]) |
130 | c0b10ca6 | Giorgos Verigakis | |
131 | c0b10ca6 | Giorgos Verigakis | if hash in os.listdir(self.trashdir): |
132 | c0b10ca6 | Giorgos Verigakis | return join(self.trashdir, hash) |
133 | c0b10ca6 | Giorgos Verigakis | |
134 | c0b10ca6 | Giorgos Verigakis | return None |
135 | d8c26d94 | Giorgos Verigakis | |
136 | c0b10ca6 | Giorgos Verigakis | def previous_hash(self, path): |
137 | c0b10ca6 | Giorgos Verigakis | """Return the hash of the file according to the previous sync with
|
138 | c0b10ca6 | Giorgos Verigakis | the server. Return DEL if not such entry exists."""
|
139 | c0b10ca6 | Giorgos Verigakis | |
140 | c0b10ca6 | Giorgos Verigakis | sql = 'SELECT hash FROM files WHERE path = ?'
|
141 | c0b10ca6 | Giorgos Verigakis | ret = self.conn.execute(sql, (path,)).fetchone()
|
142 | c0b10ca6 | Giorgos Verigakis | return ret[0] if ret else 'DEL' |
143 | 68d89033 | Giorgos Verigakis | |
144 | c0b10ca6 | Giorgos Verigakis | def remote_hash(self, path): |
145 | c0b10ca6 | Giorgos Verigakis | """Return the hash of the file according to the server"""
|
146 | c0b10ca6 | Giorgos Verigakis | |
147 | d8c26d94 | Giorgos Verigakis | try:
|
148 | c0b10ca6 | Giorgos Verigakis | meta = client.retrieve_object_metadata(self.container, path)
|
149 | d8c26d94 | Giorgos Verigakis | except Fault:
|
150 | 68d89033 | Giorgos Verigakis | return 'DEL' |
151 | 68d89033 | Giorgos Verigakis | if meta.get('content-type', None) == 'application/directory': |
152 | 68d89033 | Giorgos Verigakis | return 'DIR' |
153 | d8c26d94 | Giorgos Verigakis | else:
|
154 | 3ea35aa0 | Antony Chazapis | return meta['x-object-hash'] |
155 | c0b10ca6 | Giorgos Verigakis | |
156 | c0b10ca6 | Giorgos Verigakis | def remove_deleted_dirs(self): |
157 | c0b10ca6 | Giorgos Verigakis | for path in sorted(self.deleted_dirs, key=len, reverse=True): |
158 | c0b10ca6 | Giorgos Verigakis | os.rmdir(path) |
159 | c0b10ca6 | Giorgos Verigakis | self.deleted_dirs.remove(path)
|
160 | c0b10ca6 | Giorgos Verigakis | |
161 | c0b10ca6 | Giorgos Verigakis | def resolve_conflict(self, path, hash): |
162 | c0b10ca6 | Giorgos Verigakis | """Resolve a sync conflict by renaming the local file and downloading
|
163 | c0b10ca6 | Giorgos Verigakis | the remote one."""
|
164 | c0b10ca6 | Giorgos Verigakis | |
165 | c0b10ca6 | Giorgos Verigakis | fullpath = join(self.syncdir, path)
|
166 | c0b10ca6 | Giorgos Verigakis | resolved = fullpath + '.local'
|
167 | c0b10ca6 | Giorgos Verigakis | i = 0
|
168 | c0b10ca6 | Giorgos Verigakis | while exists(resolved):
|
169 | c0b10ca6 | Giorgos Verigakis | i += 1
|
170 | c0b10ca6 | Giorgos Verigakis | resolved = fullpath + '.local%d' % i
|
171 | c0b10ca6 | Giorgos Verigakis | |
172 | c0b10ca6 | Giorgos Verigakis | os.rename(fullpath, resolved) |
173 | c0b10ca6 | Giorgos Verigakis | self.download(path, hash) |
174 | c0b10ca6 | Giorgos Verigakis | |
175 | c0b10ca6 | Giorgos Verigakis | def rmdir(self, path): |
176 | c0b10ca6 | Giorgos Verigakis | """Remove a dir or mark for deletion if non-empty
|
177 | c0b10ca6 | Giorgos Verigakis |
|
178 | c0b10ca6 | Giorgos Verigakis | If a dir is empty delete it and check if any of its parents should be
|
179 | c0b10ca6 | Giorgos Verigakis | deleted too. Else mark it for later deletion.
|
180 | c0b10ca6 | Giorgos Verigakis | """
|
181 | c0b10ca6 | Giorgos Verigakis | |
182 | c0b10ca6 | Giorgos Verigakis | fullpath = join(self.syncdir, path)
|
183 | c0b10ca6 | Giorgos Verigakis | if not exists(fullpath): |
184 | c0b10ca6 | Giorgos Verigakis | return
|
185 | c0b10ca6 | Giorgos Verigakis | |
186 | c0b10ca6 | Giorgos Verigakis | if os.listdir(fullpath):
|
187 | c0b10ca6 | Giorgos Verigakis | # Directory not empty
|
188 | c0b10ca6 | Giorgos Verigakis | self.deleted_dirs.add(fullpath)
|
189 | c0b10ca6 | Giorgos Verigakis | return
|
190 | c0b10ca6 | Giorgos Verigakis | |
191 | c0b10ca6 | Giorgos Verigakis | os.rmdir(fullpath) |
192 | c0b10ca6 | Giorgos Verigakis | self.deleted_dirs.discard(fullpath)
|
193 | c0b10ca6 | Giorgos Verigakis | |
194 | c0b10ca6 | Giorgos Verigakis | parent = dirname(fullpath) |
195 | c0b10ca6 | Giorgos Verigakis | while parent in self.deleted_dirs: |
196 | c0b10ca6 | Giorgos Verigakis | os.rmdir(parent) |
197 | c0b10ca6 | Giorgos Verigakis | self.deleted_dirs.remove(parent)
|
198 | c0b10ca6 | Giorgos Verigakis | parent = dirname(parent) |
199 | c0b10ca6 | Giorgos Verigakis | |
200 | c0b10ca6 | Giorgos Verigakis | def save(self, path, hash): |
201 | c0b10ca6 | Giorgos Verigakis | """Save the hash value of a file. This value will be later returned
|
202 | c0b10ca6 | Giorgos Verigakis | by `previous_hash`."""
|
203 | c0b10ca6 | Giorgos Verigakis | |
204 | c0b10ca6 | Giorgos Verigakis | sql = 'INSERT OR REPLACE INTO files (path, hash) VALUES (?, ?)'
|
205 | c0b10ca6 | Giorgos Verigakis | self.conn.execute(sql, (path, hash)) |
206 | c0b10ca6 | Giorgos Verigakis | self.conn.commit()
|
207 | c0b10ca6 | Giorgos Verigakis | |
208 | c0b10ca6 | Giorgos Verigakis | def touch(self, path, now): |
209 | c0b10ca6 | Giorgos Verigakis | sql = 'UPDATE files SET timestamp = ? WHERE path = ?'
|
210 | c0b10ca6 | Giorgos Verigakis | self.conn.execute(sql, (now, path))
|
211 | c0b10ca6 | Giorgos Verigakis | self.conn.commit()
|
212 | c0b10ca6 | Giorgos Verigakis | |
213 | c0b10ca6 | Giorgos Verigakis | def trash(self, path): |
214 | c0b10ca6 | Giorgos Verigakis | """Move a file to trash or delete it if it's a directory"""
|
215 | c0b10ca6 | Giorgos Verigakis | |
216 | c0b10ca6 | Giorgos Verigakis | fullpath = join(self.syncdir, path)
|
217 | c0b10ca6 | Giorgos Verigakis | if not exists(fullpath): |
218 | c0b10ca6 | Giorgos Verigakis | return
|
219 | c0b10ca6 | Giorgos Verigakis | |
220 | c0b10ca6 | Giorgos Verigakis | if isfile(fullpath):
|
221 | c0b10ca6 | Giorgos Verigakis | hash = merkle(fullpath) |
222 | c0b10ca6 | Giorgos Verigakis | trashpath = join(self.trashdir, hash) |
223 | c0b10ca6 | Giorgos Verigakis | os.rename(fullpath, trashpath) |
224 | db3c7c26 | Antony Chazapis | else:
|
225 | c0b10ca6 | Giorgos Verigakis | self.rmdir(path)
|
226 | c0b10ca6 | Giorgos Verigakis | |
227 | c0b10ca6 | Giorgos Verigakis | def upload(self, path, hash): |
228 | c0b10ca6 | Giorgos Verigakis | fullpath = join(self.syncdir, path)
|
229 | c0b10ca6 | Giorgos Verigakis | if hash == 'DEL': |
230 | c0b10ca6 | Giorgos Verigakis | client.delete_object(self.container, path)
|
231 | c0b10ca6 | Giorgos Verigakis | elif hash == 'DIR': |
232 | c0b10ca6 | Giorgos Verigakis | client.create_directory_marker(self.container, path)
|
233 | c0b10ca6 | Giorgos Verigakis | else:
|
234 | c0b10ca6 | Giorgos Verigakis | prefix, name = split(path) |
235 | c0b10ca6 | Giorgos Verigakis | if prefix:
|
236 | c0b10ca6 | Giorgos Verigakis | prefix += '/'
|
237 | c0b10ca6 | Giorgos Verigakis | print 'Uploading %s...' % path |
238 | c0b10ca6 | Giorgos Verigakis | upload(client, fullpath, self.container, prefix, name)
|
239 | c0b10ca6 | Giorgos Verigakis | |
240 | c0b10ca6 | Giorgos Verigakis | remote = self.remote_hash(path)
|
241 | c0b10ca6 | Giorgos Verigakis | assert remote == hash, "Uploaded file does not match hash" |
242 | c0b10ca6 | Giorgos Verigakis | self.save(path, hash) |
243 | 68d89033 | Giorgos Verigakis | |
244 | 68d89033 | Giorgos Verigakis | |
245 | c0b10ca6 | Giorgos Verigakis | def sync(path, state): |
246 | c0b10ca6 | Giorgos Verigakis | previous = state.previous_hash(path) |
247 | c0b10ca6 | Giorgos Verigakis | current = state.current_hash(path) |
248 | c0b10ca6 | Giorgos Verigakis | remote = state.remote_hash(path) |
249 | d8c26d94 | Giorgos Verigakis | |
250 | c0b10ca6 | Giorgos Verigakis | if current == previous:
|
251 | c0b10ca6 | Giorgos Verigakis | # No local changes, download any remote changes
|
252 | c0b10ca6 | Giorgos Verigakis | if remote != previous:
|
253 | c0b10ca6 | Giorgos Verigakis | state.download(path, remote) |
254 | c0b10ca6 | Giorgos Verigakis | elif remote == previous:
|
255 | c0b10ca6 | Giorgos Verigakis | # No remote changes, upload any local changes
|
256 | c0b10ca6 | Giorgos Verigakis | if current != previous:
|
257 | c0b10ca6 | Giorgos Verigakis | state.upload(path, current) |
258 | d8c26d94 | Giorgos Verigakis | else:
|
259 | c0b10ca6 | Giorgos Verigakis | # Both local and remote file have changes since last sync
|
260 | c0b10ca6 | Giorgos Verigakis | if current == remote:
|
261 | c0b10ca6 | Giorgos Verigakis | state.save(path, remote) # Local and remote changes match
|
262 | c0b10ca6 | Giorgos Verigakis | else:
|
263 | c0b10ca6 | Giorgos Verigakis | state.resolve_conflict(path, remote) |
264 | 68d89033 | Giorgos Verigakis | |
265 | 68d89033 | Giorgos Verigakis | |
266 | c0b10ca6 | Giorgos Verigakis | def walk(dir, container): |
267 | c0b10ca6 | Giorgos Verigakis | """Iterates on the files of the hierarchy created by merging the files
|
268 | c0b10ca6 | Giorgos Verigakis | in `dir` and the objects in `container`."""
|
269 | c0b10ca6 | Giorgos Verigakis | |
270 | 68d89033 | Giorgos Verigakis | pending = ['']
|
271 | 68d89033 | Giorgos Verigakis | |
272 | 68d89033 | Giorgos Verigakis | while pending:
|
273 | 68d89033 | Giorgos Verigakis | dirs = set()
|
274 | 68d89033 | Giorgos Verigakis | files = set()
|
275 | c0b10ca6 | Giorgos Verigakis | root = pending.pop(0) # Depth First Traversal |
276 | c0b10ca6 | Giorgos Verigakis | if root == TRASH_DIR:
|
277 | c0b10ca6 | Giorgos Verigakis | continue
|
278 | 68d89033 | Giorgos Verigakis | if root:
|
279 | 68d89033 | Giorgos Verigakis | yield root
|
280 | 68d89033 | Giorgos Verigakis | |
281 | c0b10ca6 | Giorgos Verigakis | dirpath = join(dir, root)
|
282 | c0b10ca6 | Giorgos Verigakis | if exists(dirpath):
|
283 | 68d89033 | Giorgos Verigakis | for filename in os.listdir(dirpath): |
284 | c0b10ca6 | Giorgos Verigakis | path = join(root, filename) |
285 | c0b10ca6 | Giorgos Verigakis | if isdir(join(dir, path)): |
286 | 68d89033 | Giorgos Verigakis | dirs.add(path) |
287 | 68d89033 | Giorgos Verigakis | else:
|
288 | 68d89033 | Giorgos Verigakis | files.add(path) |
289 | 68d89033 | Giorgos Verigakis | |
290 | c0b10ca6 | Giorgos Verigakis | for object in client.list_objects(container, format='json', |
291 | c0b10ca6 | Giorgos Verigakis | prefix=root, delimiter='/'):
|
292 | 68d89033 | Giorgos Verigakis | if 'subdir' in object: |
293 | 68d89033 | Giorgos Verigakis | continue
|
294 | c0b10ca6 | Giorgos Verigakis | name = object['name'] |
295 | 68d89033 | Giorgos Verigakis | if object['content_type'] == 'application/directory': |
296 | 68d89033 | Giorgos Verigakis | dirs.add(name) |
297 | 68d89033 | Giorgos Verigakis | else:
|
298 | 68d89033 | Giorgos Verigakis | files.add(name) |
299 | 68d89033 | Giorgos Verigakis | |
300 | 68d89033 | Giorgos Verigakis | pending += sorted(dirs)
|
301 | 68d89033 | Giorgos Verigakis | for path in files: |
302 | 68d89033 | Giorgos Verigakis | yield path
|
303 | d8c26d94 | Giorgos Verigakis | |
304 | d8c26d94 | Giorgos Verigakis | |
305 | d8c26d94 | Giorgos Verigakis | def main(): |
306 | d8c26d94 | Giorgos Verigakis | if len(sys.argv) != 2: |
307 | d8c26d94 | Giorgos Verigakis | print 'syntax: %s <dir>' % sys.argv[0] |
308 | d8c26d94 | Giorgos Verigakis | sys.exit(1)
|
309 | d8c26d94 | Giorgos Verigakis | |
310 | c0b10ca6 | Giorgos Verigakis | syncdir = sys.argv[1]
|
311 | 68d89033 | Giorgos Verigakis | |
312 | c0b10ca6 | Giorgos Verigakis | _makedirs(SETTINGS_DIR) |
313 | c0b10ca6 | Giorgos Verigakis | container = os.environ.get('PITHOS_SYNC_CONTAINER', DEFAULT_CONTAINER)
|
314 | c0b10ca6 | Giorgos Verigakis | client.create_container(container) |
315 | db3c7c26 | Antony Chazapis | |
316 | c0b10ca6 | Giorgos Verigakis | state = State(syncdir, container) |
317 | 99d0e277 | Antony Chazapis | |
318 | c0b10ca6 | Giorgos Verigakis | now = int(time())
|
319 | c0b10ca6 | Giorgos Verigakis | for path in walk(syncdir, container): |
320 | 68d89033 | Giorgos Verigakis | print 'Syncing', path |
321 | c0b10ca6 | Giorgos Verigakis | sync(path, state) |
322 | c0b10ca6 | Giorgos Verigakis | state.touch(path, now) |
323 | db3c7c26 | Antony Chazapis | |
324 | c0b10ca6 | Giorgos Verigakis | state.delete_inactive(now) |
325 | c0b10ca6 | Giorgos Verigakis | state.empty_trash() |
326 | c0b10ca6 | Giorgos Verigakis | state.remove_deleted_dirs() |
327 | d8c26d94 | Giorgos Verigakis | |
328 | d8c26d94 | Giorgos Verigakis | |
329 | d8c26d94 | Giorgos Verigakis | if __name__ == '__main__': |
330 | d8c26d94 | Giorgos Verigakis | main() |