root / trunk / Pithos.Core / Agents / PollAgent.cs @ 0a9d4d18
History | View | Annotate | Download (31.3 kB)
1 | aa7ac00e | Panagiotis Kanavos | #region |
---|---|---|---|
2 | aa7ac00e | Panagiotis Kanavos | /* ----------------------------------------------------------------------- |
3 | aa7ac00e | Panagiotis Kanavos | * <copyright file="PollAgent.cs" company="GRNet"> |
4 | aa7ac00e | Panagiotis Kanavos | * |
5 | aa7ac00e | Panagiotis Kanavos | * Copyright 2011-2012 GRNET S.A. All rights reserved. |
6 | aa7ac00e | Panagiotis Kanavos | * |
7 | aa7ac00e | Panagiotis Kanavos | * Redistribution and use in source and binary forms, with or |
8 | aa7ac00e | Panagiotis Kanavos | * without modification, are permitted provided that the following |
9 | aa7ac00e | Panagiotis Kanavos | * conditions are met: |
10 | aa7ac00e | Panagiotis Kanavos | * |
11 | aa7ac00e | Panagiotis Kanavos | * 1. Redistributions of source code must retain the above |
12 | aa7ac00e | Panagiotis Kanavos | * copyright notice, this list of conditions and the following |
13 | aa7ac00e | Panagiotis Kanavos | * disclaimer. |
14 | aa7ac00e | Panagiotis Kanavos | * |
15 | aa7ac00e | Panagiotis Kanavos | * 2. Redistributions in binary form must reproduce the above |
16 | aa7ac00e | Panagiotis Kanavos | * copyright notice, this list of conditions and the following |
17 | aa7ac00e | Panagiotis Kanavos | * disclaimer in the documentation and/or other materials |
18 | aa7ac00e | Panagiotis Kanavos | * provided with the distribution. |
19 | aa7ac00e | Panagiotis Kanavos | * |
20 | aa7ac00e | Panagiotis Kanavos | * |
21 | aa7ac00e | Panagiotis Kanavos | * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
22 | aa7ac00e | Panagiotis Kanavos | * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
23 | aa7ac00e | Panagiotis Kanavos | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
24 | aa7ac00e | Panagiotis Kanavos | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
25 | aa7ac00e | Panagiotis Kanavos | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
26 | aa7ac00e | Panagiotis Kanavos | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
27 | aa7ac00e | Panagiotis Kanavos | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
28 | aa7ac00e | Panagiotis Kanavos | * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
29 | aa7ac00e | Panagiotis Kanavos | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
30 | aa7ac00e | Panagiotis Kanavos | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
31 | aa7ac00e | Panagiotis Kanavos | * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
32 | aa7ac00e | Panagiotis Kanavos | * POSSIBILITY OF SUCH DAMAGE. |
33 | aa7ac00e | Panagiotis Kanavos | * |
34 | aa7ac00e | Panagiotis Kanavos | * The views and conclusions contained in the software and |
35 | aa7ac00e | Panagiotis Kanavos | * documentation are those of the authors and should not be |
36 | aa7ac00e | Panagiotis Kanavos | * interpreted as representing official policies, either expressed |
37 | aa7ac00e | Panagiotis Kanavos | * or implied, of GRNET S.A. |
38 | aa7ac00e | Panagiotis Kanavos | * </copyright> |
39 | aa7ac00e | Panagiotis Kanavos | * ----------------------------------------------------------------------- |
40 | aa7ac00e | Panagiotis Kanavos | */ |
41 | aa7ac00e | Panagiotis Kanavos | #endregion |
42 | aa7ac00e | Panagiotis Kanavos | |
43 | aa7ac00e | Panagiotis Kanavos | using System.Collections.Concurrent; |
44 | aa7ac00e | Panagiotis Kanavos | using System.ComponentModel.Composition; |
45 | aa7ac00e | Panagiotis Kanavos | using System.Diagnostics; |
46 | aa7ac00e | Panagiotis Kanavos | using System.Diagnostics.Contracts; |
47 | aa7ac00e | Panagiotis Kanavos | using System.IO; |
48 | db8a9589 | Panagiotis Kanavos | using System.Reflection; |
49 | aa7ac00e | Panagiotis Kanavos | using System.Threading; |
50 | aa7ac00e | Panagiotis Kanavos | using System.Threading.Tasks; |
51 | aa7ac00e | Panagiotis Kanavos | using Castle.ActiveRecord; |
52 | aa7ac00e | Panagiotis Kanavos | using Pithos.Interfaces; |
53 | aa7ac00e | Panagiotis Kanavos | using Pithos.Network; |
54 | aa7ac00e | Panagiotis Kanavos | using log4net; |
55 | aa7ac00e | Panagiotis Kanavos | |
56 | aa7ac00e | Panagiotis Kanavos | namespace Pithos.Core.Agents |
57 | aa7ac00e | Panagiotis Kanavos | { |
58 | aa7ac00e | Panagiotis Kanavos | using System; |
59 | aa7ac00e | Panagiotis Kanavos | using System.Collections.Generic; |
60 | aa7ac00e | Panagiotis Kanavos | using System.Linq; |
61 | aa7ac00e | Panagiotis Kanavos | |
62 | aa7ac00e | Panagiotis Kanavos | /// <summary> |
63 | aa7ac00e | Panagiotis Kanavos | /// PollAgent periodically polls the server to detect object changes. The agent retrieves a listing of all |
64 | aa7ac00e | Panagiotis Kanavos | /// objects and compares it with a previously cached version to detect differences. |
65 | aa7ac00e | Panagiotis Kanavos | /// New files are downloaded, missing files are deleted from the local file system and common files are compared |
66 | aa7ac00e | Panagiotis Kanavos | /// to determine the appropriate action |
67 | aa7ac00e | Panagiotis Kanavos | /// </summary> |
68 | aa7ac00e | Panagiotis Kanavos | [Export] |
69 | aa7ac00e | Panagiotis Kanavos | public class PollAgent |
70 | aa7ac00e | Panagiotis Kanavos | { |
71 | db8a9589 | Panagiotis Kanavos | private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); |
72 | aa7ac00e | Panagiotis Kanavos | |
73 | aa7ac00e | Panagiotis Kanavos | [System.ComponentModel.Composition.Import] |
74 | aa7ac00e | Panagiotis Kanavos | public IStatusKeeper StatusKeeper { get; set; } |
75 | aa7ac00e | Panagiotis Kanavos | |
76 | aa7ac00e | Panagiotis Kanavos | [System.ComponentModel.Composition.Import] |
77 | aa7ac00e | Panagiotis Kanavos | public IPithosSettings Settings { get; set; } |
78 | aa7ac00e | Panagiotis Kanavos | |
79 | aa7ac00e | Panagiotis Kanavos | [System.ComponentModel.Composition.Import] |
80 | aa7ac00e | Panagiotis Kanavos | public NetworkAgent NetworkAgent { get; set; } |
81 | aa7ac00e | Panagiotis Kanavos | |
82 | aa7ac00e | Panagiotis Kanavos | public IStatusNotification StatusNotification { get; set; } |
83 | aa7ac00e | Panagiotis Kanavos | |
84 | aa7ac00e | Panagiotis Kanavos | private bool _firstPoll = true; |
85 | aa7ac00e | Panagiotis Kanavos | |
86 | aa7ac00e | Panagiotis Kanavos | //The Sync Event signals a manual synchronisation |
87 | aa7ac00e | Panagiotis Kanavos | private readonly AsyncManualResetEvent _syncEvent = new AsyncManualResetEvent(); |
88 | aa7ac00e | Panagiotis Kanavos | |
89 | ec1a1baf | Panagiotis Kanavos | private readonly ConcurrentDictionary<string, DateTime> _lastSeen = new ConcurrentDictionary<string, DateTime>(); |
90 | 1a41e9ec | pkanavos | private readonly ConcurrentDictionary<Uri, AccountInfo> _accounts = new ConcurrentDictionary<Uri,AccountInfo>(); |
91 | aa7ac00e | Panagiotis Kanavos | |
92 | aa7ac00e | Panagiotis Kanavos | |
93 | aa7ac00e | Panagiotis Kanavos | /// <summary> |
94 | aa7ac00e | Panagiotis Kanavos | /// Start a manual synchronization |
95 | aa7ac00e | Panagiotis Kanavos | /// </summary> |
96 | aa7ac00e | Panagiotis Kanavos | public void SynchNow() |
97 | aa7ac00e | Panagiotis Kanavos | { |
98 | aa7ac00e | Panagiotis Kanavos | _syncEvent.Set(); |
99 | aa7ac00e | Panagiotis Kanavos | } |
100 | aa7ac00e | Panagiotis Kanavos | |
101 | fec5da06 | Panagiotis Kanavos | /// <summary> |
102 | fec5da06 | Panagiotis Kanavos | /// Remote files are polled periodically. Any changes are processed |
103 | fec5da06 | Panagiotis Kanavos | /// </summary> |
104 | fec5da06 | Panagiotis Kanavos | /// <param name="since"></param> |
105 | fec5da06 | Panagiotis Kanavos | /// <returns></returns> |
106 | aa7ac00e | Panagiotis Kanavos | public async Task PollRemoteFiles(DateTime? since = null) |
107 | aa7ac00e | Panagiotis Kanavos | { |
108 | 174bbb6e | Panagiotis Kanavos | if (Log.IsDebugEnabled) |
109 | 174bbb6e | Panagiotis Kanavos | Log.DebugFormat("Polling changes after [{0}]",since); |
110 | aa7ac00e | Panagiotis Kanavos | |
111 | 174bbb6e | Panagiotis Kanavos | Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!"); |
112 | 174bbb6e | Panagiotis Kanavos | |
113 | aa7ac00e | Panagiotis Kanavos | |
114 | ec1a1baf | Panagiotis Kanavos | using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts")) |
115 | aa7ac00e | Panagiotis Kanavos | { |
116 | aa7ac00e | Panagiotis Kanavos | //If this poll fails, we will retry with the same since value |
117 | aa7ac00e | Panagiotis Kanavos | var nextSince = since; |
118 | aa7ac00e | Panagiotis Kanavos | try |
119 | aa7ac00e | Panagiotis Kanavos | { |
120 | 174bbb6e | Panagiotis Kanavos | UpdateStatus(PithosStatus.PollSyncing); |
121 | 174bbb6e | Panagiotis Kanavos | |
122 | ec1a1baf | Panagiotis Kanavos | var tasks = from accountInfo in _accounts.Values |
123 | aa7ac00e | Panagiotis Kanavos | select ProcessAccountFiles(accountInfo, since); |
124 | aa7ac00e | Panagiotis Kanavos | |
125 | 6f03d6e1 | Panagiotis Kanavos | var nextTimes=await TaskEx.WhenAll(tasks.ToList()); |
126 | aa7ac00e | Panagiotis Kanavos | |
127 | aa7ac00e | Panagiotis Kanavos | _firstPoll = false; |
128 | aa7ac00e | Panagiotis Kanavos | //Reschedule the poll with the current timestamp as a "since" value |
129 | 6f03d6e1 | Panagiotis Kanavos | |
130 | 6f03d6e1 | Panagiotis Kanavos | if (nextTimes.Length>0) |
131 | 6f03d6e1 | Panagiotis Kanavos | nextSince = nextTimes.Min(); |
132 | 6f03d6e1 | Panagiotis Kanavos | if (Log.IsDebugEnabled) |
133 | 6f03d6e1 | Panagiotis Kanavos | Log.DebugFormat("Next Poll at [{0}]",nextSince); |
134 | aa7ac00e | Panagiotis Kanavos | } |
135 | aa7ac00e | Panagiotis Kanavos | catch (Exception ex) |
136 | aa7ac00e | Panagiotis Kanavos | { |
137 | aa7ac00e | Panagiotis Kanavos | Log.ErrorFormat("Error while processing accounts\r\n{0}", ex); |
138 | aa7ac00e | Panagiotis Kanavos | //In case of failure retry with the same "since" value |
139 | aa7ac00e | Panagiotis Kanavos | } |
140 | aa7ac00e | Panagiotis Kanavos | |
141 | 174bbb6e | Panagiotis Kanavos | UpdateStatus(PithosStatus.PollComplete); |
142 | fec5da06 | Panagiotis Kanavos | //The multiple try blocks are required because we can't have an await call |
143 | fec5da06 | Panagiotis Kanavos | //inside a finally block |
144 | fec5da06 | Panagiotis Kanavos | //TODO: Find a more elegant solution for reschedulling in the event of an exception |
145 | fec5da06 | Panagiotis Kanavos | try |
146 | fec5da06 | Panagiotis Kanavos | { |
147 | fec5da06 | Panagiotis Kanavos | //Wait for the polling interval to pass or the Sync event to be signalled |
148 | fec5da06 | Panagiotis Kanavos | nextSince = await WaitForScheduledOrManualPoll(nextSince); |
149 | fec5da06 | Panagiotis Kanavos | } |
150 | fec5da06 | Panagiotis Kanavos | finally |
151 | fec5da06 | Panagiotis Kanavos | { |
152 | fec5da06 | Panagiotis Kanavos | //Ensure polling is scheduled even in case of error |
153 | fec5da06 | Panagiotis Kanavos | TaskEx.Run(() => PollRemoteFiles(nextSince)); |
154 | fec5da06 | Panagiotis Kanavos | } |
155 | aa7ac00e | Panagiotis Kanavos | } |
156 | aa7ac00e | Panagiotis Kanavos | } |
157 | aa7ac00e | Panagiotis Kanavos | |
158 | aa7ac00e | Panagiotis Kanavos | /// <summary> |
159 | aa7ac00e | Panagiotis Kanavos | /// Wait for the polling period to expire or a manual sync request |
160 | aa7ac00e | Panagiotis Kanavos | /// </summary> |
161 | aa7ac00e | Panagiotis Kanavos | /// <param name="since"></param> |
162 | aa7ac00e | Panagiotis Kanavos | /// <returns></returns> |
163 | aa7ac00e | Panagiotis Kanavos | private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since) |
164 | aa7ac00e | Panagiotis Kanavos | { |
165 | aa7ac00e | Panagiotis Kanavos | var sync = _syncEvent.WaitAsync(); |
166 | aa7ac00e | Panagiotis Kanavos | var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval), NetworkAgent.CancellationToken); |
167 | aa7ac00e | Panagiotis Kanavos | var signaledTask = await TaskEx.WhenAny(sync, wait); |
168 | aa7ac00e | Panagiotis Kanavos | |
169 | aa7ac00e | Panagiotis Kanavos | //Wait for network processing to finish before polling |
170 | 38ac43a6 | Panagiotis Kanavos | var pauseTask=NetworkAgent.ProceedEvent.WaitAsync(); |
171 | aa7ac00e | Panagiotis Kanavos | await TaskEx.WhenAll(signaledTask, pauseTask); |
172 | aa7ac00e | Panagiotis Kanavos | |
173 | aa7ac00e | Panagiotis Kanavos | //If polling is signalled by SynchNow, ignore the since tag |
174 | aa7ac00e | Panagiotis Kanavos | if (sync.IsCompleted) |
175 | aa7ac00e | Panagiotis Kanavos | { |
176 | aa7ac00e | Panagiotis Kanavos | //TODO: Must convert to AutoReset |
177 | aa7ac00e | Panagiotis Kanavos | _syncEvent.Reset(); |
178 | aa7ac00e | Panagiotis Kanavos | return null; |
179 | aa7ac00e | Panagiotis Kanavos | } |
180 | aa7ac00e | Panagiotis Kanavos | return since; |
181 | aa7ac00e | Panagiotis Kanavos | } |
182 | aa7ac00e | Panagiotis Kanavos | |
183 | 6f03d6e1 | Panagiotis Kanavos | public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, DateTime? since = null) |
184 | aa7ac00e | Panagiotis Kanavos | { |
185 | aa7ac00e | Panagiotis Kanavos | if (accountInfo == null) |
186 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentNullException("accountInfo"); |
187 | aa7ac00e | Panagiotis Kanavos | if (String.IsNullOrWhiteSpace(accountInfo.AccountPath)) |
188 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo"); |
189 | aa7ac00e | Panagiotis Kanavos | Contract.EndContractBlock(); |
190 | aa7ac00e | Panagiotis Kanavos | |
191 | aa7ac00e | Panagiotis Kanavos | |
192 | dccd340f | Panagiotis Kanavos | using (ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName)) |
193 | aa7ac00e | Panagiotis Kanavos | { |
194 | 6f03d6e1 | Panagiotis Kanavos | |
195 | aa7ac00e | Panagiotis Kanavos | await NetworkAgent.GetDeleteAwaiter(); |
196 | aa7ac00e | Panagiotis Kanavos | |
197 | aa7ac00e | Panagiotis Kanavos | Log.Info("Scheduled"); |
198 | aa7ac00e | Panagiotis Kanavos | var client = new CloudFilesClient(accountInfo); |
199 | aa7ac00e | Panagiotis Kanavos | |
200 | 99e6329f | Panagiotis Kanavos | //We don't need to check the trash container |
201 | ec1a1baf | Panagiotis Kanavos | var containers = client.ListContainers(accountInfo.UserName) |
202 | ec1a1baf | Panagiotis Kanavos | .Where(c=>c.Name!="trash") |
203 | ec1a1baf | Panagiotis Kanavos | .ToList(); |
204 | aa7ac00e | Panagiotis Kanavos | |
205 | aa7ac00e | Panagiotis Kanavos | |
206 | aa7ac00e | Panagiotis Kanavos | CreateContainerFolders(accountInfo, containers); |
207 | aa7ac00e | Panagiotis Kanavos | |
208 | 6f03d6e1 | Panagiotis Kanavos | //The nextSince time fallback time is the same as the current. |
209 | 6f03d6e1 | Panagiotis Kanavos | //If polling succeeds, the next Since time will be the smallest of the maximum modification times |
210 | 6f03d6e1 | Panagiotis Kanavos | //of the shared and account objects |
211 | 6f03d6e1 | Panagiotis Kanavos | var nextSince = since; |
212 | 6f03d6e1 | Panagiotis Kanavos | |
213 | aa7ac00e | Panagiotis Kanavos | try |
214 | aa7ac00e | Panagiotis Kanavos | { |
215 | 759bd3c4 | Panagiotis Kanavos | //Wait for any deletions to finish |
216 | aa7ac00e | Panagiotis Kanavos | await NetworkAgent.GetDeleteAwaiter(); |
217 | aa7ac00e | Panagiotis Kanavos | //Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted |
218 | aa7ac00e | Panagiotis Kanavos | //than delete a file that was created while we were executing the poll |
219 | aa7ac00e | Panagiotis Kanavos | |
220 | aa7ac00e | Panagiotis Kanavos | //Get the list of server objects changed since the last check |
221 | aa7ac00e | Panagiotis Kanavos | //The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step |
222 | aa7ac00e | Panagiotis Kanavos | var listObjects = (from container in containers |
223 | aa7ac00e | Panagiotis Kanavos | select Task<IList<ObjectInfo>>.Factory.StartNew(_ => |
224 | aa7ac00e | Panagiotis Kanavos | client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList(); |
225 | 99e6329f | Panagiotis Kanavos | |
226 | 99e6329f | Panagiotis Kanavos | var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => |
227 | 99e6329f | Panagiotis Kanavos | client.ListSharedObjects(since), "shared"); |
228 | aa7ac00e | Panagiotis Kanavos | listObjects.Add(listShared); |
229 | aa7ac00e | Panagiotis Kanavos | var listTasks = await Task.Factory.WhenAll(listObjects.ToArray()); |
230 | aa7ac00e | Panagiotis Kanavos | |
231 | dccd340f | Panagiotis Kanavos | using (ThreadContext.Stacks["SCHEDULE"].Push("Process Results")) |
232 | aa7ac00e | Panagiotis Kanavos | { |
233 | aa7ac00e | Panagiotis Kanavos | var dict = listTasks.ToDictionary(t => t.AsyncState); |
234 | aa7ac00e | Panagiotis Kanavos | |
235 | aa7ac00e | Panagiotis Kanavos | //Get all non-trash objects. Remember, the container name is stored in AsyncState |
236 | 6f03d6e1 | Panagiotis Kanavos | var remoteObjects = (from objectList in listTasks |
237 | aa7ac00e | Panagiotis Kanavos | where (string)objectList.AsyncState != "trash" |
238 | aa7ac00e | Panagiotis Kanavos | from obj in objectList.Result |
239 | 6f03d6e1 | Panagiotis Kanavos | select obj).ToList(); |
240 | 6f03d6e1 | Panagiotis Kanavos | |
241 | 6f03d6e1 | Panagiotis Kanavos | //Get the latest remote object modification date, only if it is after |
242 | 6f03d6e1 | Panagiotis Kanavos | //the original since date |
243 | 6f03d6e1 | Panagiotis Kanavos | nextSince = GetLatestDateAfter(nextSince, remoteObjects); |
244 | aa7ac00e | Panagiotis Kanavos | |
245 | aa7ac00e | Panagiotis Kanavos | var sharedObjects = dict["shared"].Result; |
246 | 6f03d6e1 | Panagiotis Kanavos | nextSince = GetLatestDateBefore(nextSince, sharedObjects); |
247 | aa7ac00e | Panagiotis Kanavos | |
248 | aa7ac00e | Panagiotis Kanavos | //DON'T process trashed files |
249 | aa7ac00e | Panagiotis Kanavos | //If some files are deleted and added again to a folder, they will be deleted |
250 | aa7ac00e | Panagiotis Kanavos | //even though they are new. |
251 | aa7ac00e | Panagiotis Kanavos | //We would have to check file dates and hashes to ensure that a trashed file |
252 | aa7ac00e | Panagiotis Kanavos | //can be deleted safely from the local hard drive. |
253 | aa7ac00e | Panagiotis Kanavos | /* |
254 | aa7ac00e | Panagiotis Kanavos | //Items with the same name, hash may be both in the container and the trash |
255 | aa7ac00e | Panagiotis Kanavos | //Don't delete items that exist in the container |
256 | aa7ac00e | Panagiotis Kanavos | var realTrash = from trash in trashObjects |
257 | aa7ac00e | Panagiotis Kanavos | where |
258 | aa7ac00e | Panagiotis Kanavos | !remoteObjects.Any( |
259 | aa7ac00e | Panagiotis Kanavos | info => info.Name == trash.Name && info.Hash == trash.Hash) |
260 | aa7ac00e | Panagiotis Kanavos | select trash; |
261 | aa7ac00e | Panagiotis Kanavos | ProcessTrashedFiles(accountInfo, realTrash); |
262 | aa7ac00e | Panagiotis Kanavos | */ |
263 | aa7ac00e | Panagiotis Kanavos | |
264 | aa7ac00e | Panagiotis Kanavos | var cleanRemotes = (from info in remoteObjects.Union(sharedObjects) |
265 | 92f18b56 | Panagiotis Kanavos | let name = info.Name??"" |
266 | aa7ac00e | Panagiotis Kanavos | where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) && |
267 | aa7ac00e | Panagiotis Kanavos | !name.StartsWith(FolderConstants.CacheFolder + "/", |
268 | aa7ac00e | Panagiotis Kanavos | StringComparison.InvariantCultureIgnoreCase) |
269 | aa7ac00e | Panagiotis Kanavos | select info).ToList(); |
270 | aa7ac00e | Panagiotis Kanavos | |
271 | bc27bb7e | Panagiotis Kanavos | if (_firstPoll) |
272 | bc27bb7e | Panagiotis Kanavos | StatusKeeper.CleanupOrphanStates(); |
273 | bc27bb7e | Panagiotis Kanavos | StatusKeeper.CleanupStaleStates(accountInfo, cleanRemotes); |
274 | bc27bb7e | Panagiotis Kanavos | |
275 | aa7ac00e | Panagiotis Kanavos | var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes); |
276 | aa7ac00e | Panagiotis Kanavos | |
277 | ebc37b0d | pkanavos | var filterUris = SelectiveUris[accountInfo.AccountKey]; |
278 | ebc37b0d | pkanavos | |
279 | ebc37b0d | pkanavos | ProcessDeletedFiles(accountInfo, differencer.Deleted.FilterDirectlyBelow(filterUris)); |
280 | 759bd3c4 | Panagiotis Kanavos | |
281 | 759bd3c4 | Panagiotis Kanavos | // @@@ NEED To add previous state here as well, To compare with previous hash |
282 | 759bd3c4 | Panagiotis Kanavos | |
283 | 759bd3c4 | Panagiotis Kanavos | |
284 | aa7ac00e | Panagiotis Kanavos | |
285 | aa7ac00e | Panagiotis Kanavos | //Create a list of actions from the remote files |
286 | ebc37b0d | pkanavos | |
287 | ebc37b0d | pkanavos | var allActions = MovesToActions(accountInfo,differencer.Moved.FilterDirectlyBelow(filterUris)) |
288 | b666b39a | Panagiotis Kanavos | .Union( |
289 | ebc37b0d | pkanavos | ChangesToActions(accountInfo, differencer.Changed.FilterDirectlyBelow(filterUris))) |
290 | aa7ac00e | Panagiotis Kanavos | .Union( |
291 | ebc37b0d | pkanavos | CreatesToActions(accountInfo, differencer.Created.FilterDirectlyBelow(filterUris))); |
292 | aa7ac00e | Panagiotis Kanavos | |
293 | aa7ac00e | Panagiotis Kanavos | //And remove those that are already being processed by the agent |
294 | aa7ac00e | Panagiotis Kanavos | var distinctActions = allActions |
295 | dccd340f | Panagiotis Kanavos | .Except(NetworkAgent.GetEnumerable(), new LocalFileComparer()) |
296 | aa7ac00e | Panagiotis Kanavos | .ToList(); |
297 | aa7ac00e | Panagiotis Kanavos | |
298 | aa7ac00e | Panagiotis Kanavos | //Queue all the actions |
299 | aa7ac00e | Panagiotis Kanavos | foreach (var message in distinctActions) |
300 | aa7ac00e | Panagiotis Kanavos | { |
301 | aa7ac00e | Panagiotis Kanavos | NetworkAgent.Post(message); |
302 | aa7ac00e | Panagiotis Kanavos | } |
303 | aa7ac00e | Panagiotis Kanavos | |
304 | aa7ac00e | Panagiotis Kanavos | Log.Info("[LISTENER] End Processing"); |
305 | aa7ac00e | Panagiotis Kanavos | } |
306 | aa7ac00e | Panagiotis Kanavos | } |
307 | aa7ac00e | Panagiotis Kanavos | catch (Exception ex) |
308 | aa7ac00e | Panagiotis Kanavos | { |
309 | aa7ac00e | Panagiotis Kanavos | Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex); |
310 | 6f03d6e1 | Panagiotis Kanavos | return nextSince; |
311 | aa7ac00e | Panagiotis Kanavos | } |
312 | aa7ac00e | Panagiotis Kanavos | |
313 | aa7ac00e | Panagiotis Kanavos | Log.Info("[LISTENER] Finished"); |
314 | 6f03d6e1 | Panagiotis Kanavos | return nextSince; |
315 | aa7ac00e | Panagiotis Kanavos | } |
316 | aa7ac00e | Panagiotis Kanavos | } |
317 | aa7ac00e | Panagiotis Kanavos | |
318 | 6f03d6e1 | Panagiotis Kanavos | /// <summary> |
319 | 6f03d6e1 | Panagiotis Kanavos | /// Returns the latest LastModified date from the list of objects, but only if it is before |
320 | 6f03d6e1 | Panagiotis Kanavos | /// than the threshold value |
321 | 6f03d6e1 | Panagiotis Kanavos | /// </summary> |
322 | 6f03d6e1 | Panagiotis Kanavos | /// <param name="threshold"></param> |
323 | 6f03d6e1 | Panagiotis Kanavos | /// <param name="cloudObjects"></param> |
324 | 6f03d6e1 | Panagiotis Kanavos | /// <returns></returns> |
325 | 6f03d6e1 | Panagiotis Kanavos | private static DateTime? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects) |
326 | 6f03d6e1 | Panagiotis Kanavos | { |
327 | 6f03d6e1 | Panagiotis Kanavos | DateTime? maxDate = null; |
328 | 6f03d6e1 | Panagiotis Kanavos | if (cloudObjects!=null && cloudObjects.Count > 0) |
329 | 6f03d6e1 | Panagiotis Kanavos | maxDate = cloudObjects.Max(obj => obj.Last_Modified); |
330 | 6f03d6e1 | Panagiotis Kanavos | if (maxDate == null || maxDate == DateTime.MinValue) |
331 | 6f03d6e1 | Panagiotis Kanavos | return threshold; |
332 | 6f03d6e1 | Panagiotis Kanavos | if (threshold == null || threshold == DateTime.MinValue || threshold > maxDate) |
333 | 6f03d6e1 | Panagiotis Kanavos | return maxDate; |
334 | 6f03d6e1 | Panagiotis Kanavos | return threshold; |
335 | 6f03d6e1 | Panagiotis Kanavos | } |
336 | 6f03d6e1 | Panagiotis Kanavos | |
337 | 6f03d6e1 | Panagiotis Kanavos | /// <summary> |
338 | 6f03d6e1 | Panagiotis Kanavos | /// Returns the latest LastModified date from the list of objects, but only if it is after |
339 | 6f03d6e1 | Panagiotis Kanavos | /// the threshold value |
340 | 6f03d6e1 | Panagiotis Kanavos | /// </summary> |
341 | 6f03d6e1 | Panagiotis Kanavos | /// <param name="threshold"></param> |
342 | 6f03d6e1 | Panagiotis Kanavos | /// <param name="cloudObjects"></param> |
343 | 6f03d6e1 | Panagiotis Kanavos | /// <returns></returns> |
344 | 6f03d6e1 | Panagiotis Kanavos | private static DateTime? GetLatestDateAfter(DateTime? threshold, IList<ObjectInfo> cloudObjects) |
345 | 6f03d6e1 | Panagiotis Kanavos | { |
346 | 6f03d6e1 | Panagiotis Kanavos | DateTime? maxDate = null; |
347 | 6f03d6e1 | Panagiotis Kanavos | if (cloudObjects!=null && cloudObjects.Count > 0) |
348 | 6f03d6e1 | Panagiotis Kanavos | maxDate = cloudObjects.Max(obj => obj.Last_Modified); |
349 | 6f03d6e1 | Panagiotis Kanavos | if (maxDate == null || maxDate == DateTime.MinValue) |
350 | 6f03d6e1 | Panagiotis Kanavos | return threshold; |
351 | 6f03d6e1 | Panagiotis Kanavos | if (threshold == null || threshold == DateTime.MinValue || threshold < maxDate) |
352 | 6f03d6e1 | Panagiotis Kanavos | return maxDate; |
353 | 6f03d6e1 | Panagiotis Kanavos | return threshold; |
354 | 6f03d6e1 | Panagiotis Kanavos | } |
355 | 6f03d6e1 | Panagiotis Kanavos | |
356 | ec1a1baf | Panagiotis Kanavos | readonly AccountsDifferencer _differencer = new AccountsDifferencer(); |
357 | ebc37b0d | pkanavos | private Dictionary<Uri, List<Uri>> _selectiveUris = new Dictionary<Uri, List<Uri>>(); |
358 | aa7ac00e | Panagiotis Kanavos | |
359 | aa7ac00e | Panagiotis Kanavos | /// <summary> |
360 | aa7ac00e | Panagiotis Kanavos | /// Deletes local files that are not found in the list of cloud files |
361 | aa7ac00e | Panagiotis Kanavos | /// </summary> |
362 | aa7ac00e | Panagiotis Kanavos | /// <param name="accountInfo"></param> |
363 | aa7ac00e | Panagiotis Kanavos | /// <param name="cloudFiles"></param> |
364 | ec1a1baf | Panagiotis Kanavos | private void ProcessDeletedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles) |
365 | aa7ac00e | Panagiotis Kanavos | { |
366 | aa7ac00e | Panagiotis Kanavos | if (accountInfo == null) |
367 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentNullException("accountInfo"); |
368 | aa7ac00e | Panagiotis Kanavos | if (String.IsNullOrWhiteSpace(accountInfo.AccountPath)) |
369 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo"); |
370 | aa7ac00e | Panagiotis Kanavos | if (cloudFiles == null) |
371 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentNullException("cloudFiles"); |
372 | aa7ac00e | Panagiotis Kanavos | Contract.EndContractBlock(); |
373 | aa7ac00e | Panagiotis Kanavos | |
374 | aa7ac00e | Panagiotis Kanavos | //On the first run |
375 | aa7ac00e | Panagiotis Kanavos | if (_firstPoll) |
376 | aa7ac00e | Panagiotis Kanavos | { |
377 | aa7ac00e | Panagiotis Kanavos | //Only consider files that are not being modified, ie they are in the Unchanged state |
378 | aa7ac00e | Panagiotis Kanavos | var deleteCandidates = FileState.Queryable.Where(state => |
379 | aa7ac00e | Panagiotis Kanavos | state.FilePath.StartsWith(accountInfo.AccountPath) |
380 | aa7ac00e | Panagiotis Kanavos | && state.FileStatus == FileStatus.Unchanged).ToList(); |
381 | aa7ac00e | Panagiotis Kanavos | |
382 | aa7ac00e | Panagiotis Kanavos | |
383 | aa7ac00e | Panagiotis Kanavos | //TODO: filesToDelete must take into account the Others container |
384 | aa7ac00e | Panagiotis Kanavos | var filesToDelete = (from deleteCandidate in deleteCandidates |
385 | aa7ac00e | Panagiotis Kanavos | let localFile = FileInfoExtensions.FromPath(deleteCandidate.FilePath) |
386 | aa7ac00e | Panagiotis Kanavos | let relativeFilePath = localFile.AsRelativeTo(accountInfo.AccountPath) |
387 | aa7ac00e | Panagiotis Kanavos | where |
388 | aa7ac00e | Panagiotis Kanavos | !cloudFiles.Any(r => r.RelativeUrlToFilePath(accountInfo.UserName) == relativeFilePath) |
389 | aa7ac00e | Panagiotis Kanavos | select localFile).ToList(); |
390 | aa7ac00e | Panagiotis Kanavos | |
391 | aa7ac00e | Panagiotis Kanavos | |
392 | aa7ac00e | Panagiotis Kanavos | |
393 | aa7ac00e | Panagiotis Kanavos | //Set the status of missing files to Conflict |
394 | aa7ac00e | Panagiotis Kanavos | foreach (var item in filesToDelete) |
395 | aa7ac00e | Panagiotis Kanavos | { |
396 | aa7ac00e | Panagiotis Kanavos | //Try to acquire a gate on the file, to take into account files that have been dequeued |
397 | aa7ac00e | Panagiotis Kanavos | //and are being processed |
398 | aa7ac00e | Panagiotis Kanavos | using (var gate = NetworkGate.Acquire(item.FullName, NetworkOperation.Deleting)) |
399 | aa7ac00e | Panagiotis Kanavos | { |
400 | aa7ac00e | Panagiotis Kanavos | if (gate.Failed) |
401 | aa7ac00e | Panagiotis Kanavos | continue; |
402 | 0a9d4d18 | pkanavos | StatusKeeper.SetFileState(item.FullName, FileStatus.Conflict, FileOverlayStatus.Deleted,"Local file missing from server"); |
403 | aa7ac00e | Panagiotis Kanavos | } |
404 | aa7ac00e | Panagiotis Kanavos | } |
405 | aa7ac00e | Panagiotis Kanavos | UpdateStatus(PithosStatus.HasConflicts); |
406 | aa7ac00e | Panagiotis Kanavos | StatusNotification.NotifyConflicts(filesToDelete, String.Format("{0} local files are missing from Pithos, possibly because they were deleted", filesToDelete.Count)); |
407 | aa7ac00e | Panagiotis Kanavos | StatusNotification.NotifyForFiles(filesToDelete, String.Format("{0} files were deleted", filesToDelete.Count), TraceLevel.Info); |
408 | aa7ac00e | Panagiotis Kanavos | } |
409 | aa7ac00e | Panagiotis Kanavos | else |
410 | aa7ac00e | Panagiotis Kanavos | { |
411 | aa7ac00e | Panagiotis Kanavos | var deletedFiles = new List<FileSystemInfo>(); |
412 | aa7ac00e | Panagiotis Kanavos | foreach (var objectInfo in cloudFiles) |
413 | aa7ac00e | Panagiotis Kanavos | { |
414 | 81c5c310 | pkanavos | if (Log.IsDebugEnabled) |
415 | 81c5c310 | pkanavos | Log.DebugFormat("Handle deleted [{0}]",objectInfo.Uri); |
416 | aa7ac00e | Panagiotis Kanavos | var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName); |
417 | fbbbe99b | Panagiotis Kanavos | var item = FileAgent.GetFileAgent(accountInfo).GetFileSystemInfo(relativePath); |
418 | 81c5c310 | pkanavos | if (Log.IsDebugEnabled) |
419 | 81c5c310 | pkanavos | Log.DebugFormat("Will delete [{0}] for [{1}]", item.FullName,objectInfo.Uri); |
420 | aa7ac00e | Panagiotis Kanavos | if (item.Exists) |
421 | aa7ac00e | Panagiotis Kanavos | { |
422 | aa7ac00e | Panagiotis Kanavos | if ((item.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) |
423 | aa7ac00e | Panagiotis Kanavos | { |
424 | aa7ac00e | Panagiotis Kanavos | item.Attributes = item.Attributes & ~FileAttributes.ReadOnly; |
425 | aa7ac00e | Panagiotis Kanavos | |
426 | aa7ac00e | Panagiotis Kanavos | } |
427 | 81c5c310 | pkanavos | |
428 | 81c5c310 | pkanavos | |
429 | 81c5c310 | pkanavos | Log.DebugFormat("Deleting {0}", item.FullName); |
430 | 81c5c310 | pkanavos | |
431 | 6bcdd8e2 | Panagiotis Kanavos | var directory = item as DirectoryInfo; |
432 | 6bcdd8e2 | Panagiotis Kanavos | if (directory!=null) |
433 | 6bcdd8e2 | Panagiotis Kanavos | directory.Delete(true); |
434 | 6bcdd8e2 | Panagiotis Kanavos | else |
435 | 6bcdd8e2 | Panagiotis Kanavos | item.Delete(); |
436 | 6bcdd8e2 | Panagiotis Kanavos | Log.DebugFormat("Deleted [{0}] for [{1}]", item.FullName, objectInfo.Uri); |
437 | aa7ac00e | Panagiotis Kanavos | DateTime lastDate; |
438 | aa7ac00e | Panagiotis Kanavos | _lastSeen.TryRemove(item.FullName, out lastDate); |
439 | aa7ac00e | Panagiotis Kanavos | deletedFiles.Add(item); |
440 | aa7ac00e | Panagiotis Kanavos | } |
441 | 0a9d4d18 | pkanavos | StatusKeeper.SetFileState(item.FullName, FileStatus.Deleted, FileOverlayStatus.Deleted, "File Deleted"); |
442 | aa7ac00e | Panagiotis Kanavos | } |
443 | 6bcdd8e2 | Panagiotis Kanavos | Log.InfoFormat("[{0}] files were deleted",deletedFiles.Count); |
444 | aa7ac00e | Panagiotis Kanavos | StatusNotification.NotifyForFiles(deletedFiles, String.Format("{0} files were deleted", deletedFiles.Count), TraceLevel.Info); |
445 | aa7ac00e | Panagiotis Kanavos | } |
446 | aa7ac00e | Panagiotis Kanavos | |
447 | aa7ac00e | Panagiotis Kanavos | } |
448 | aa7ac00e | Panagiotis Kanavos | |
449 | b666b39a | Panagiotis Kanavos | /// <summary> |
450 | b666b39a | Panagiotis Kanavos | /// Creates a Sync action for each changed server file |
451 | b666b39a | Panagiotis Kanavos | /// </summary> |
452 | b666b39a | Panagiotis Kanavos | /// <param name="accountInfo"></param> |
453 | b666b39a | Panagiotis Kanavos | /// <param name="changes"></param> |
454 | b666b39a | Panagiotis Kanavos | /// <returns></returns> |
455 | aa7ac00e | Panagiotis Kanavos | private IEnumerable<CloudAction> ChangesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> changes) |
456 | aa7ac00e | Panagiotis Kanavos | { |
457 | aa7ac00e | Panagiotis Kanavos | if (changes == null) |
458 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentNullException(); |
459 | aa7ac00e | Panagiotis Kanavos | Contract.EndContractBlock(); |
460 | fbbbe99b | Panagiotis Kanavos | var fileAgent = FileAgent.GetFileAgent(accountInfo); |
461 | aa7ac00e | Panagiotis Kanavos | |
462 | aa7ac00e | Panagiotis Kanavos | //In order to avoid multiple iterations over the files, we iterate only once |
463 | aa7ac00e | Panagiotis Kanavos | //over the remote files |
464 | aa7ac00e | Panagiotis Kanavos | foreach (var objectInfo in changes) |
465 | aa7ac00e | Panagiotis Kanavos | { |
466 | aa7ac00e | Panagiotis Kanavos | var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName); |
467 | b666b39a | Panagiotis Kanavos | //If a directory object already exists, we may need to sync it |
468 | aa7ac00e | Panagiotis Kanavos | if (fileAgent.Exists(relativePath)) |
469 | aa7ac00e | Panagiotis Kanavos | { |
470 | aa7ac00e | Panagiotis Kanavos | var localFile = fileAgent.GetFileSystemInfo(relativePath); |
471 | b666b39a | Panagiotis Kanavos | //We don't need to sync directories |
472 | aa7ac00e | Panagiotis Kanavos | if (objectInfo.Content_Type == @"application/directory" && localFile is DirectoryInfo) |
473 | aa7ac00e | Panagiotis Kanavos | continue; |
474 | aa7ac00e | Panagiotis Kanavos | using (new SessionScope(FlushAction.Never)) |
475 | aa7ac00e | Panagiotis Kanavos | { |
476 | aa7ac00e | Panagiotis Kanavos | var state = StatusKeeper.GetStateByFilePath(localFile.FullName); |
477 | aa7ac00e | Panagiotis Kanavos | _lastSeen[localFile.FullName] = DateTime.Now; |
478 | aa7ac00e | Panagiotis Kanavos | //Common files should be checked on a per-case basis to detect differences, which is newer |
479 | aa7ac00e | Panagiotis Kanavos | |
480 | aa7ac00e | Panagiotis Kanavos | yield return new CloudAction(accountInfo, CloudActionType.MustSynch, |
481 | aa7ac00e | Panagiotis Kanavos | localFile, objectInfo, state, accountInfo.BlockSize, |
482 | aa7ac00e | Panagiotis Kanavos | accountInfo.BlockHash); |
483 | aa7ac00e | Panagiotis Kanavos | } |
484 | aa7ac00e | Panagiotis Kanavos | } |
485 | aa7ac00e | Panagiotis Kanavos | else |
486 | aa7ac00e | Panagiotis Kanavos | { |
487 | aa7ac00e | Panagiotis Kanavos | //Remote files should be downloaded |
488 | aa7ac00e | Panagiotis Kanavos | yield return new CloudDownloadAction(accountInfo, objectInfo); |
489 | aa7ac00e | Panagiotis Kanavos | } |
490 | aa7ac00e | Panagiotis Kanavos | } |
491 | aa7ac00e | Panagiotis Kanavos | } |
492 | aa7ac00e | Panagiotis Kanavos | |
493 | b666b39a | Panagiotis Kanavos | /// <summary> |
494 | b666b39a | Panagiotis Kanavos | /// Creates a Local Move action for each moved server file |
495 | b666b39a | Panagiotis Kanavos | /// </summary> |
496 | b666b39a | Panagiotis Kanavos | /// <param name="accountInfo"></param> |
497 | b666b39a | Panagiotis Kanavos | /// <param name="moves"></param> |
498 | b666b39a | Panagiotis Kanavos | /// <returns></returns> |
499 | b666b39a | Panagiotis Kanavos | private IEnumerable<CloudAction> MovesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> moves) |
500 | b666b39a | Panagiotis Kanavos | { |
501 | b666b39a | Panagiotis Kanavos | if (moves == null) |
502 | b666b39a | Panagiotis Kanavos | throw new ArgumentNullException(); |
503 | b666b39a | Panagiotis Kanavos | Contract.EndContractBlock(); |
504 | b666b39a | Panagiotis Kanavos | var fileAgent = FileAgent.GetFileAgent(accountInfo); |
505 | b666b39a | Panagiotis Kanavos | |
506 | b666b39a | Panagiotis Kanavos | //In order to avoid multiple iterations over the files, we iterate only once |
507 | b666b39a | Panagiotis Kanavos | //over the remote files |
508 | b666b39a | Panagiotis Kanavos | foreach (var objectInfo in moves) |
509 | b666b39a | Panagiotis Kanavos | { |
510 | b666b39a | Panagiotis Kanavos | var previousRelativepath = objectInfo.Previous.RelativeUrlToFilePath(accountInfo.UserName); |
511 | b666b39a | Panagiotis Kanavos | //If the previous file already exists, we can execute a Move operation |
512 | b666b39a | Panagiotis Kanavos | if (fileAgent.Exists(previousRelativepath)) |
513 | b666b39a | Panagiotis Kanavos | { |
514 | b666b39a | Panagiotis Kanavos | var previousFile = fileAgent.GetFileSystemInfo(previousRelativepath); |
515 | b666b39a | Panagiotis Kanavos | using (new SessionScope(FlushAction.Never)) |
516 | b666b39a | Panagiotis Kanavos | { |
517 | b666b39a | Panagiotis Kanavos | var state = StatusKeeper.GetStateByFilePath(previousFile.FullName); |
518 | b666b39a | Panagiotis Kanavos | _lastSeen[previousFile.FullName] = DateTime.Now; |
519 | b666b39a | Panagiotis Kanavos | |
520 | b666b39a | Panagiotis Kanavos | //For each moved object we need to move both the local file and update |
521 | b666b39a | Panagiotis Kanavos | yield return new CloudAction(accountInfo, CloudActionType.RenameLocal, |
522 | b666b39a | Panagiotis Kanavos | previousFile, objectInfo, state, accountInfo.BlockSize, |
523 | b666b39a | Panagiotis Kanavos | accountInfo.BlockHash); |
524 | b666b39a | Panagiotis Kanavos | //For modified files, we need to download the changes as well |
525 | b666b39a | Panagiotis Kanavos | if (objectInfo.Hash!=objectInfo.PreviousHash) |
526 | b666b39a | Panagiotis Kanavos | yield return new CloudDownloadAction(accountInfo,objectInfo); |
527 | b666b39a | Panagiotis Kanavos | } |
528 | b666b39a | Panagiotis Kanavos | } |
529 | b666b39a | Panagiotis Kanavos | //If the previous file does not exist, we need to download it in the new location |
530 | b666b39a | Panagiotis Kanavos | else |
531 | b666b39a | Panagiotis Kanavos | { |
532 | b666b39a | Panagiotis Kanavos | //Remote files should be downloaded |
533 | b666b39a | Panagiotis Kanavos | yield return new CloudDownloadAction(accountInfo, objectInfo); |
534 | b666b39a | Panagiotis Kanavos | } |
535 | b666b39a | Panagiotis Kanavos | } |
536 | b666b39a | Panagiotis Kanavos | } |
537 | b666b39a | Panagiotis Kanavos | |
538 | b666b39a | Panagiotis Kanavos | |
539 | b666b39a | Panagiotis Kanavos | /// <summary> |
540 | b666b39a | Panagiotis Kanavos | /// Creates a download action for each new server file |
541 | b666b39a | Panagiotis Kanavos | /// </summary> |
542 | b666b39a | Panagiotis Kanavos | /// <param name="accountInfo"></param> |
543 | b666b39a | Panagiotis Kanavos | /// <param name="creates"></param> |
544 | b666b39a | Panagiotis Kanavos | /// <returns></returns> |
545 | aa7ac00e | Panagiotis Kanavos | private IEnumerable<CloudAction> CreatesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> creates) |
546 | aa7ac00e | Panagiotis Kanavos | { |
547 | aa7ac00e | Panagiotis Kanavos | if (creates == null) |
548 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentNullException(); |
549 | aa7ac00e | Panagiotis Kanavos | Contract.EndContractBlock(); |
550 | fbbbe99b | Panagiotis Kanavos | var fileAgent = FileAgent.GetFileAgent(accountInfo); |
551 | aa7ac00e | Panagiotis Kanavos | |
552 | aa7ac00e | Panagiotis Kanavos | //In order to avoid multiple iterations over the files, we iterate only once |
553 | aa7ac00e | Panagiotis Kanavos | //over the remote files |
554 | aa7ac00e | Panagiotis Kanavos | foreach (var objectInfo in creates) |
555 | aa7ac00e | Panagiotis Kanavos | { |
556 | dccd340f | Panagiotis Kanavos | if (Log.IsDebugEnabled) |
557 | dccd340f | Panagiotis Kanavos | Log.DebugFormat("[NEW INFO] {0}",objectInfo.Uri); |
558 | dccd340f | Panagiotis Kanavos | |
559 | aa7ac00e | Panagiotis Kanavos | var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName); |
560 | 0a9d4d18 | pkanavos | |
561 | 0a9d4d18 | pkanavos | //If the object already exists, we should check before uploading or downloading |
562 | aa7ac00e | Panagiotis Kanavos | if (fileAgent.Exists(relativePath)) |
563 | aa7ac00e | Panagiotis Kanavos | { |
564 | 0a9d4d18 | pkanavos | var localFile= fileAgent.GetFileSystemInfo(relativePath); |
565 | 0a9d4d18 | pkanavos | var state = StatusKeeper.GetStateByFilePath(localFile.WithProperCapitalization().FullName); |
566 | 0a9d4d18 | pkanavos | yield return new CloudAction(accountInfo, CloudActionType.MustSynch, |
567 | 0a9d4d18 | pkanavos | localFile, objectInfo, state, accountInfo.BlockSize, |
568 | 0a9d4d18 | pkanavos | accountInfo.BlockHash); |
569 | aa7ac00e | Panagiotis Kanavos | } |
570 | aa7ac00e | Panagiotis Kanavos | else |
571 | aa7ac00e | Panagiotis Kanavos | { |
572 | aa7ac00e | Panagiotis Kanavos | //Remote files should be downloaded |
573 | aa7ac00e | Panagiotis Kanavos | yield return new CloudDownloadAction(accountInfo, objectInfo); |
574 | aa7ac00e | Panagiotis Kanavos | } |
575 | 0a9d4d18 | pkanavos | |
576 | aa7ac00e | Panagiotis Kanavos | } |
577 | aa7ac00e | Panagiotis Kanavos | } |
578 | aa7ac00e | Panagiotis Kanavos | |
579 | fec5da06 | Panagiotis Kanavos | /// <summary> |
580 | fec5da06 | Panagiotis Kanavos | /// Notify the UI to update the visual status |
581 | fec5da06 | Panagiotis Kanavos | /// </summary> |
582 | fec5da06 | Panagiotis Kanavos | /// <param name="status"></param> |
583 | aa7ac00e | Panagiotis Kanavos | private void UpdateStatus(PithosStatus status) |
584 | aa7ac00e | Panagiotis Kanavos | { |
585 | fec5da06 | Panagiotis Kanavos | try |
586 | fec5da06 | Panagiotis Kanavos | { |
587 | 174bbb6e | Panagiotis Kanavos | StatusNotification.SetPithosStatus(status); |
588 | 174bbb6e | Panagiotis Kanavos | //StatusNotification.Notify(new Notification()); |
589 | fec5da06 | Panagiotis Kanavos | } |
590 | fec5da06 | Panagiotis Kanavos | catch (Exception exc) |
591 | fec5da06 | Panagiotis Kanavos | { |
592 | fec5da06 | Panagiotis Kanavos | //Failure is not critical, just log it |
593 | fec5da06 | Panagiotis Kanavos | Log.Warn("Error while updating status", exc); |
594 | fec5da06 | Panagiotis Kanavos | } |
595 | aa7ac00e | Panagiotis Kanavos | } |
596 | aa7ac00e | Panagiotis Kanavos | |
597 | aa7ac00e | Panagiotis Kanavos | private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers) |
598 | aa7ac00e | Panagiotis Kanavos | { |
599 | aa7ac00e | Panagiotis Kanavos | var containerPaths = from container in containers |
600 | aa7ac00e | Panagiotis Kanavos | let containerPath = Path.Combine(accountInfo.AccountPath, container.Name) |
601 | aa7ac00e | Panagiotis Kanavos | where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath) |
602 | aa7ac00e | Panagiotis Kanavos | select containerPath; |
603 | aa7ac00e | Panagiotis Kanavos | |
604 | aa7ac00e | Panagiotis Kanavos | foreach (var path in containerPaths) |
605 | aa7ac00e | Panagiotis Kanavos | { |
606 | aa7ac00e | Panagiotis Kanavos | Directory.CreateDirectory(path); |
607 | aa7ac00e | Panagiotis Kanavos | } |
608 | aa7ac00e | Panagiotis Kanavos | } |
609 | 759bd3c4 | Panagiotis Kanavos | |
610 | ebc37b0d | pkanavos | public void SetSyncUris(Uri accountKey, Uri[] uris) |
611 | fec5da06 | Panagiotis Kanavos | { |
612 | ebc37b0d | pkanavos | SelectiveUris[accountKey]=uris.ToList(); |
613 | 759bd3c4 | Panagiotis Kanavos | } |
614 | 759bd3c4 | Panagiotis Kanavos | |
615 | ebc37b0d | pkanavos | protected Dictionary<Uri,List<Uri>> SelectiveUris |
616 | 759bd3c4 | Panagiotis Kanavos | { |
617 | 759bd3c4 | Panagiotis Kanavos | get { return _selectiveUris;} |
618 | 759bd3c4 | Panagiotis Kanavos | set { _selectiveUris = value; } |
619 | 759bd3c4 | Panagiotis Kanavos | } |
620 | fec5da06 | Panagiotis Kanavos | |
621 | fec5da06 | Panagiotis Kanavos | public void AddAccount(AccountInfo accountInfo) |
622 | fec5da06 | Panagiotis Kanavos | { |
623 | ec1a1baf | Panagiotis Kanavos | //Avoid adding a duplicate accountInfo |
624 | 1a41e9ec | pkanavos | _accounts.TryAdd(accountInfo.AccountKey, accountInfo); |
625 | ec1a1baf | Panagiotis Kanavos | } |
626 | ec1a1baf | Panagiotis Kanavos | |
627 | ec1a1baf | Panagiotis Kanavos | public void RemoveAccount(AccountInfo accountInfo) |
628 | ec1a1baf | Panagiotis Kanavos | { |
629 | ec1a1baf | Panagiotis Kanavos | AccountInfo account; |
630 | 1a41e9ec | pkanavos | _accounts.TryRemove(accountInfo.AccountKey, out account); |
631 | dccd340f | Panagiotis Kanavos | SnapshotDifferencer differencer; |
632 | 1a41e9ec | pkanavos | _differencer.Differencers.TryRemove(accountInfo.AccountKey, out differencer); |
633 | fec5da06 | Panagiotis Kanavos | } |
634 | 026a6c5a | Panagiotis Kanavos | |
635 | 026a6c5a | Panagiotis Kanavos | public void SetSelectivePaths(AccountInfo accountInfo,Uri[] added, Uri[] removed) |
636 | 026a6c5a | Panagiotis Kanavos | { |
637 | 026a6c5a | Panagiotis Kanavos | AbortRemovedPaths(accountInfo,removed); |
638 | 026a6c5a | Panagiotis Kanavos | DownloadNewPaths(accountInfo,added); |
639 | 026a6c5a | Panagiotis Kanavos | } |
640 | 026a6c5a | Panagiotis Kanavos | |
641 | 026a6c5a | Panagiotis Kanavos | private void DownloadNewPaths(AccountInfo accountInfo, Uri[] added) |
642 | 026a6c5a | Panagiotis Kanavos | { |
643 | 026a6c5a | Panagiotis Kanavos | var client = new CloudFilesClient(accountInfo); |
644 | 026a6c5a | Panagiotis Kanavos | foreach (var folderUri in added) |
645 | 026a6c5a | Panagiotis Kanavos | { |
646 | 026a6c5a | Panagiotis Kanavos | string account; |
647 | 026a6c5a | Panagiotis Kanavos | string container; |
648 | 026a6c5a | Panagiotis Kanavos | var segmentsCount = folderUri.Segments.Length; |
649 | 026a6c5a | Panagiotis Kanavos | if (segmentsCount < 3) |
650 | 026a6c5a | Panagiotis Kanavos | continue; |
651 | 026a6c5a | Panagiotis Kanavos | if (segmentsCount==3) |
652 | 026a6c5a | Panagiotis Kanavos | { |
653 | 026a6c5a | Panagiotis Kanavos | account = folderUri.Segments[1].TrimEnd('/'); |
654 | 026a6c5a | Panagiotis Kanavos | container = folderUri.Segments[2].TrimEnd('/'); |
655 | 026a6c5a | Panagiotis Kanavos | } |
656 | 026a6c5a | Panagiotis Kanavos | else |
657 | 026a6c5a | Panagiotis Kanavos | { |
658 | 026a6c5a | Panagiotis Kanavos | account = folderUri.Segments[2].TrimEnd('/'); |
659 | 026a6c5a | Panagiotis Kanavos | container = folderUri.Segments[3].TrimEnd('/'); |
660 | 026a6c5a | Panagiotis Kanavos | } |
661 | 026a6c5a | Panagiotis Kanavos | IList<ObjectInfo> items; |
662 | 026a6c5a | Panagiotis Kanavos | if(segmentsCount>3) |
663 | 026a6c5a | Panagiotis Kanavos | { |
664 | 026a6c5a | Panagiotis Kanavos | var folder =String.Join("", folderUri.Segments.Splice(4)); |
665 | 026a6c5a | Panagiotis Kanavos | items = client.ListObjects(account, container, folder); |
666 | 026a6c5a | Panagiotis Kanavos | } |
667 | 026a6c5a | Panagiotis Kanavos | else |
668 | 026a6c5a | Panagiotis Kanavos | { |
669 | 026a6c5a | Panagiotis Kanavos | items = client.ListObjects(account, container); |
670 | 026a6c5a | Panagiotis Kanavos | } |
671 | 026a6c5a | Panagiotis Kanavos | var actions=CreatesToActions(accountInfo, items); |
672 | 026a6c5a | Panagiotis Kanavos | foreach (var action in actions) |
673 | 026a6c5a | Panagiotis Kanavos | { |
674 | 026a6c5a | Panagiotis Kanavos | NetworkAgent.Post(action); |
675 | 026a6c5a | Panagiotis Kanavos | } |
676 | 026a6c5a | Panagiotis Kanavos | } |
677 | 026a6c5a | Panagiotis Kanavos | |
678 | 026a6c5a | Panagiotis Kanavos | //Need to get a listing of each of the URLs, then post them to the NetworkAgent |
679 | 026a6c5a | Panagiotis Kanavos | //CreatesToActions(accountInfo,) |
680 | 026a6c5a | Panagiotis Kanavos | |
681 | 026a6c5a | Panagiotis Kanavos | /* NetworkAgent.Post();*/ |
682 | 026a6c5a | Panagiotis Kanavos | } |
683 | 026a6c5a | Panagiotis Kanavos | |
684 | 026a6c5a | Panagiotis Kanavos | private void AbortRemovedPaths(AccountInfo accountInfo, Uri[] removed) |
685 | 026a6c5a | Panagiotis Kanavos | { |
686 | 026a6c5a | Panagiotis Kanavos | /*this.NetworkAgent.*/ |
687 | 026a6c5a | Panagiotis Kanavos | } |
688 | aa7ac00e | Panagiotis Kanavos | } |
689 | aa7ac00e | Panagiotis Kanavos | } |