root / trunk / Pithos.Core / Agents / PollAgent.cs @ dccd340f
History | View | Annotate | Download (28.6 kB)
1 | aa7ac00e | Panagiotis Kanavos | #region |
---|---|---|---|
2 | aa7ac00e | Panagiotis Kanavos | /* ----------------------------------------------------------------------- |
3 | aa7ac00e | Panagiotis Kanavos | * <copyright file="PollAgent.cs" company="GRNet"> |
4 | aa7ac00e | Panagiotis Kanavos | * |
5 | aa7ac00e | Panagiotis Kanavos | * Copyright 2011-2012 GRNET S.A. All rights reserved. |
6 | aa7ac00e | Panagiotis Kanavos | * |
7 | aa7ac00e | Panagiotis Kanavos | * Redistribution and use in source and binary forms, with or |
8 | aa7ac00e | Panagiotis Kanavos | * without modification, are permitted provided that the following |
9 | aa7ac00e | Panagiotis Kanavos | * conditions are met: |
10 | aa7ac00e | Panagiotis Kanavos | * |
11 | aa7ac00e | Panagiotis Kanavos | * 1. Redistributions of source code must retain the above |
12 | aa7ac00e | Panagiotis Kanavos | * copyright notice, this list of conditions and the following |
13 | aa7ac00e | Panagiotis Kanavos | * disclaimer. |
14 | aa7ac00e | Panagiotis Kanavos | * |
15 | aa7ac00e | Panagiotis Kanavos | * 2. Redistributions in binary form must reproduce the above |
16 | aa7ac00e | Panagiotis Kanavos | * copyright notice, this list of conditions and the following |
17 | aa7ac00e | Panagiotis Kanavos | * disclaimer in the documentation and/or other materials |
18 | aa7ac00e | Panagiotis Kanavos | * provided with the distribution. |
19 | aa7ac00e | Panagiotis Kanavos | * |
20 | aa7ac00e | Panagiotis Kanavos | * |
21 | aa7ac00e | Panagiotis Kanavos | * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
22 | aa7ac00e | Panagiotis Kanavos | * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
23 | aa7ac00e | Panagiotis Kanavos | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
24 | aa7ac00e | Panagiotis Kanavos | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
25 | aa7ac00e | Panagiotis Kanavos | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
26 | aa7ac00e | Panagiotis Kanavos | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
27 | aa7ac00e | Panagiotis Kanavos | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
28 | aa7ac00e | Panagiotis Kanavos | * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
29 | aa7ac00e | Panagiotis Kanavos | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
30 | aa7ac00e | Panagiotis Kanavos | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
31 | aa7ac00e | Panagiotis Kanavos | * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
32 | aa7ac00e | Panagiotis Kanavos | * POSSIBILITY OF SUCH DAMAGE. |
33 | aa7ac00e | Panagiotis Kanavos | * |
34 | aa7ac00e | Panagiotis Kanavos | * The views and conclusions contained in the software and |
35 | aa7ac00e | Panagiotis Kanavos | * documentation are those of the authors and should not be |
36 | aa7ac00e | Panagiotis Kanavos | * interpreted as representing official policies, either expressed |
37 | aa7ac00e | Panagiotis Kanavos | * or implied, of GRNET S.A. |
38 | aa7ac00e | Panagiotis Kanavos | * </copyright> |
39 | aa7ac00e | Panagiotis Kanavos | * ----------------------------------------------------------------------- |
40 | aa7ac00e | Panagiotis Kanavos | */ |
41 | aa7ac00e | Panagiotis Kanavos | #endregion |
42 | aa7ac00e | Panagiotis Kanavos | |
43 | aa7ac00e | Panagiotis Kanavos | using System.Collections.Concurrent; |
44 | aa7ac00e | Panagiotis Kanavos | using System.ComponentModel.Composition; |
45 | aa7ac00e | Panagiotis Kanavos | using System.Diagnostics; |
46 | aa7ac00e | Panagiotis Kanavos | using System.Diagnostics.Contracts; |
47 | aa7ac00e | Panagiotis Kanavos | using System.IO; |
48 | db8a9589 | Panagiotis Kanavos | using System.Reflection; |
49 | aa7ac00e | Panagiotis Kanavos | using System.Threading; |
50 | aa7ac00e | Panagiotis Kanavos | using System.Threading.Tasks; |
51 | aa7ac00e | Panagiotis Kanavos | using Castle.ActiveRecord; |
52 | aa7ac00e | Panagiotis Kanavos | using Pithos.Interfaces; |
53 | aa7ac00e | Panagiotis Kanavos | using Pithos.Network; |
54 | aa7ac00e | Panagiotis Kanavos | using log4net; |
55 | aa7ac00e | Panagiotis Kanavos | |
56 | aa7ac00e | Panagiotis Kanavos | namespace Pithos.Core.Agents |
57 | aa7ac00e | Panagiotis Kanavos | { |
58 | aa7ac00e | Panagiotis Kanavos | using System; |
59 | aa7ac00e | Panagiotis Kanavos | using System.Collections.Generic; |
60 | aa7ac00e | Panagiotis Kanavos | using System.Linq; |
61 | aa7ac00e | Panagiotis Kanavos | |
62 | aa7ac00e | Panagiotis Kanavos | /// <summary> |
63 | aa7ac00e | Panagiotis Kanavos | /// PollAgent periodically polls the server to detect object changes. The agent retrieves a listing of all |
64 | aa7ac00e | Panagiotis Kanavos | /// objects and compares it with a previously cached version to detect differences. |
65 | aa7ac00e | Panagiotis Kanavos | /// New files are downloaded, missing files are deleted from the local file system and common files are compared |
66 | aa7ac00e | Panagiotis Kanavos | /// to determine the appropriate action |
67 | aa7ac00e | Panagiotis Kanavos | /// </summary> |
68 | aa7ac00e | Panagiotis Kanavos | [Export] |
69 | aa7ac00e | Panagiotis Kanavos | public class PollAgent |
70 | aa7ac00e | Panagiotis Kanavos | { |
71 | db8a9589 | Panagiotis Kanavos | private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); |
72 | aa7ac00e | Panagiotis Kanavos | |
73 | aa7ac00e | Panagiotis Kanavos | [System.ComponentModel.Composition.Import] |
74 | aa7ac00e | Panagiotis Kanavos | public IStatusKeeper StatusKeeper { get; set; } |
75 | aa7ac00e | Panagiotis Kanavos | |
76 | aa7ac00e | Panagiotis Kanavos | [System.ComponentModel.Composition.Import] |
77 | aa7ac00e | Panagiotis Kanavos | public IPithosSettings Settings { get; set; } |
78 | aa7ac00e | Panagiotis Kanavos | |
79 | aa7ac00e | Panagiotis Kanavos | [System.ComponentModel.Composition.Import] |
80 | aa7ac00e | Panagiotis Kanavos | public NetworkAgent NetworkAgent { get; set; } |
81 | aa7ac00e | Panagiotis Kanavos | |
82 | aa7ac00e | Panagiotis Kanavos | public IStatusNotification StatusNotification { get; set; } |
83 | aa7ac00e | Panagiotis Kanavos | |
84 | aa7ac00e | Panagiotis Kanavos | private bool _firstPoll = true; |
85 | aa7ac00e | Panagiotis Kanavos | |
86 | aa7ac00e | Panagiotis Kanavos | //The Sync Event signals a manual synchronisation |
87 | aa7ac00e | Panagiotis Kanavos | private readonly AsyncManualResetEvent _syncEvent = new AsyncManualResetEvent(); |
88 | aa7ac00e | Panagiotis Kanavos | |
89 | ec1a1baf | Panagiotis Kanavos | private readonly ConcurrentDictionary<string, DateTime> _lastSeen = new ConcurrentDictionary<string, DateTime>(); |
90 | ec1a1baf | Panagiotis Kanavos | private readonly ConcurrentDictionary<string, AccountInfo> _accounts = new ConcurrentDictionary<string,AccountInfo>(); |
91 | aa7ac00e | Panagiotis Kanavos | |
92 | aa7ac00e | Panagiotis Kanavos | |
93 | aa7ac00e | Panagiotis Kanavos | /// <summary> |
94 | aa7ac00e | Panagiotis Kanavos | /// Start a manual synchronization |
95 | aa7ac00e | Panagiotis Kanavos | /// </summary> |
96 | aa7ac00e | Panagiotis Kanavos | public void SynchNow() |
97 | aa7ac00e | Panagiotis Kanavos | { |
98 | aa7ac00e | Panagiotis Kanavos | _syncEvent.Set(); |
99 | aa7ac00e | Panagiotis Kanavos | } |
100 | aa7ac00e | Panagiotis Kanavos | |
101 | fec5da06 | Panagiotis Kanavos | /// <summary> |
102 | fec5da06 | Panagiotis Kanavos | /// Remote files are polled periodically. Any changes are processed |
103 | fec5da06 | Panagiotis Kanavos | /// </summary> |
104 | fec5da06 | Panagiotis Kanavos | /// <param name="since"></param> |
105 | fec5da06 | Panagiotis Kanavos | /// <returns></returns> |
106 | aa7ac00e | Panagiotis Kanavos | public async Task PollRemoteFiles(DateTime? since = null) |
107 | aa7ac00e | Panagiotis Kanavos | { |
108 | 174bbb6e | Panagiotis Kanavos | if (Log.IsDebugEnabled) |
109 | 174bbb6e | Panagiotis Kanavos | Log.DebugFormat("Polling changes after [{0}]",since); |
110 | aa7ac00e | Panagiotis Kanavos | |
111 | 174bbb6e | Panagiotis Kanavos | Debug.Assert(Thread.CurrentThread.IsBackground, "Polling Ended up in the main thread!"); |
112 | 174bbb6e | Panagiotis Kanavos | |
113 | aa7ac00e | Panagiotis Kanavos | |
114 | ec1a1baf | Panagiotis Kanavos | using (ThreadContext.Stacks["Retrieve Remote"].Push("All accounts")) |
115 | aa7ac00e | Panagiotis Kanavos | { |
116 | aa7ac00e | Panagiotis Kanavos | //If this poll fails, we will retry with the same since value |
117 | aa7ac00e | Panagiotis Kanavos | var nextSince = since; |
118 | aa7ac00e | Panagiotis Kanavos | try |
119 | aa7ac00e | Panagiotis Kanavos | { |
120 | 174bbb6e | Panagiotis Kanavos | UpdateStatus(PithosStatus.PollSyncing); |
121 | 174bbb6e | Panagiotis Kanavos | |
122 | ec1a1baf | Panagiotis Kanavos | var tasks = from accountInfo in _accounts.Values |
123 | aa7ac00e | Panagiotis Kanavos | select ProcessAccountFiles(accountInfo, since); |
124 | aa7ac00e | Panagiotis Kanavos | |
125 | 6f03d6e1 | Panagiotis Kanavos | var nextTimes=await TaskEx.WhenAll(tasks.ToList()); |
126 | aa7ac00e | Panagiotis Kanavos | |
127 | aa7ac00e | Panagiotis Kanavos | _firstPoll = false; |
128 | aa7ac00e | Panagiotis Kanavos | //Reschedule the poll with the current timestamp as a "since" value |
129 | 6f03d6e1 | Panagiotis Kanavos | |
130 | 6f03d6e1 | Panagiotis Kanavos | if (nextTimes.Length>0) |
131 | 6f03d6e1 | Panagiotis Kanavos | nextSince = nextTimes.Min(); |
132 | 6f03d6e1 | Panagiotis Kanavos | if (Log.IsDebugEnabled) |
133 | 6f03d6e1 | Panagiotis Kanavos | Log.DebugFormat("Next Poll at [{0}]",nextSince); |
134 | aa7ac00e | Panagiotis Kanavos | } |
135 | aa7ac00e | Panagiotis Kanavos | catch (Exception ex) |
136 | aa7ac00e | Panagiotis Kanavos | { |
137 | aa7ac00e | Panagiotis Kanavos | Log.ErrorFormat("Error while processing accounts\r\n{0}", ex); |
138 | aa7ac00e | Panagiotis Kanavos | //In case of failure retry with the same "since" value |
139 | aa7ac00e | Panagiotis Kanavos | } |
140 | aa7ac00e | Panagiotis Kanavos | |
141 | 174bbb6e | Panagiotis Kanavos | UpdateStatus(PithosStatus.PollComplete); |
142 | fec5da06 | Panagiotis Kanavos | //The multiple try blocks are required because we can't have an await call |
143 | fec5da06 | Panagiotis Kanavos | //inside a finally block |
144 | fec5da06 | Panagiotis Kanavos | //TODO: Find a more elegant solution for reschedulling in the event of an exception |
145 | fec5da06 | Panagiotis Kanavos | try |
146 | fec5da06 | Panagiotis Kanavos | { |
147 | fec5da06 | Panagiotis Kanavos | //Wait for the polling interval to pass or the Sync event to be signalled |
148 | fec5da06 | Panagiotis Kanavos | nextSince = await WaitForScheduledOrManualPoll(nextSince); |
149 | fec5da06 | Panagiotis Kanavos | } |
150 | fec5da06 | Panagiotis Kanavos | finally |
151 | fec5da06 | Panagiotis Kanavos | { |
152 | fec5da06 | Panagiotis Kanavos | //Ensure polling is scheduled even in case of error |
153 | fec5da06 | Panagiotis Kanavos | TaskEx.Run(() => PollRemoteFiles(nextSince)); |
154 | fec5da06 | Panagiotis Kanavos | } |
155 | aa7ac00e | Panagiotis Kanavos | } |
156 | aa7ac00e | Panagiotis Kanavos | } |
157 | aa7ac00e | Panagiotis Kanavos | |
158 | aa7ac00e | Panagiotis Kanavos | /// <summary> |
159 | aa7ac00e | Panagiotis Kanavos | /// Wait for the polling period to expire or a manual sync request |
160 | aa7ac00e | Panagiotis Kanavos | /// </summary> |
161 | aa7ac00e | Panagiotis Kanavos | /// <param name="since"></param> |
162 | aa7ac00e | Panagiotis Kanavos | /// <returns></returns> |
163 | aa7ac00e | Panagiotis Kanavos | private async Task<DateTime?> WaitForScheduledOrManualPoll(DateTime? since) |
164 | aa7ac00e | Panagiotis Kanavos | { |
165 | aa7ac00e | Panagiotis Kanavos | var sync = _syncEvent.WaitAsync(); |
166 | aa7ac00e | Panagiotis Kanavos | var wait = TaskEx.Delay(TimeSpan.FromSeconds(Settings.PollingInterval), NetworkAgent.CancellationToken); |
167 | aa7ac00e | Panagiotis Kanavos | var signaledTask = await TaskEx.WhenAny(sync, wait); |
168 | aa7ac00e | Panagiotis Kanavos | |
169 | aa7ac00e | Panagiotis Kanavos | //Wait for network processing to finish before polling |
170 | 38ac43a6 | Panagiotis Kanavos | var pauseTask=NetworkAgent.ProceedEvent.WaitAsync(); |
171 | aa7ac00e | Panagiotis Kanavos | await TaskEx.WhenAll(signaledTask, pauseTask); |
172 | aa7ac00e | Panagiotis Kanavos | |
173 | aa7ac00e | Panagiotis Kanavos | //If polling is signalled by SynchNow, ignore the since tag |
174 | aa7ac00e | Panagiotis Kanavos | if (sync.IsCompleted) |
175 | aa7ac00e | Panagiotis Kanavos | { |
176 | aa7ac00e | Panagiotis Kanavos | //TODO: Must convert to AutoReset |
177 | aa7ac00e | Panagiotis Kanavos | _syncEvent.Reset(); |
178 | aa7ac00e | Panagiotis Kanavos | return null; |
179 | aa7ac00e | Panagiotis Kanavos | } |
180 | aa7ac00e | Panagiotis Kanavos | return since; |
181 | aa7ac00e | Panagiotis Kanavos | } |
182 | aa7ac00e | Panagiotis Kanavos | |
183 | 6f03d6e1 | Panagiotis Kanavos | public async Task<DateTime?> ProcessAccountFiles(AccountInfo accountInfo, DateTime? since = null) |
184 | aa7ac00e | Panagiotis Kanavos | { |
185 | aa7ac00e | Panagiotis Kanavos | if (accountInfo == null) |
186 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentNullException("accountInfo"); |
187 | aa7ac00e | Panagiotis Kanavos | if (String.IsNullOrWhiteSpace(accountInfo.AccountPath)) |
188 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo"); |
189 | aa7ac00e | Panagiotis Kanavos | Contract.EndContractBlock(); |
190 | aa7ac00e | Panagiotis Kanavos | |
191 | aa7ac00e | Panagiotis Kanavos | |
192 | dccd340f | Panagiotis Kanavos | using (ThreadContext.Stacks["Retrieve Remote"].Push(accountInfo.UserName)) |
193 | aa7ac00e | Panagiotis Kanavos | { |
194 | 6f03d6e1 | Panagiotis Kanavos | |
195 | aa7ac00e | Panagiotis Kanavos | await NetworkAgent.GetDeleteAwaiter(); |
196 | aa7ac00e | Panagiotis Kanavos | |
197 | aa7ac00e | Panagiotis Kanavos | Log.Info("Scheduled"); |
198 | aa7ac00e | Panagiotis Kanavos | var client = new CloudFilesClient(accountInfo); |
199 | aa7ac00e | Panagiotis Kanavos | |
200 | 99e6329f | Panagiotis Kanavos | //We don't need to check the trash container |
201 | ec1a1baf | Panagiotis Kanavos | var containers = client.ListContainers(accountInfo.UserName) |
202 | ec1a1baf | Panagiotis Kanavos | .Where(c=>c.Name!="trash") |
203 | ec1a1baf | Panagiotis Kanavos | .ToList(); |
204 | aa7ac00e | Panagiotis Kanavos | |
205 | aa7ac00e | Panagiotis Kanavos | |
206 | aa7ac00e | Panagiotis Kanavos | CreateContainerFolders(accountInfo, containers); |
207 | aa7ac00e | Panagiotis Kanavos | |
208 | 6f03d6e1 | Panagiotis Kanavos | //The nextSince time fallback time is the same as the current. |
209 | 6f03d6e1 | Panagiotis Kanavos | //If polling succeeds, the next Since time will be the smallest of the maximum modification times |
210 | 6f03d6e1 | Panagiotis Kanavos | //of the shared and account objects |
211 | 6f03d6e1 | Panagiotis Kanavos | var nextSince = since; |
212 | 6f03d6e1 | Panagiotis Kanavos | |
213 | aa7ac00e | Panagiotis Kanavos | try |
214 | aa7ac00e | Panagiotis Kanavos | { |
215 | 759bd3c4 | Panagiotis Kanavos | //Wait for any deletions to finish |
216 | aa7ac00e | Panagiotis Kanavos | await NetworkAgent.GetDeleteAwaiter(); |
217 | aa7ac00e | Panagiotis Kanavos | //Get the poll time now. We may miss some deletions but it's better to keep a file that was deleted |
218 | aa7ac00e | Panagiotis Kanavos | //than delete a file that was created while we were executing the poll |
219 | aa7ac00e | Panagiotis Kanavos | |
220 | aa7ac00e | Panagiotis Kanavos | //Get the list of server objects changed since the last check |
221 | aa7ac00e | Panagiotis Kanavos | //The name of the container is passed as state in order to create a dictionary of tasks in a subsequent step |
222 | aa7ac00e | Panagiotis Kanavos | var listObjects = (from container in containers |
223 | aa7ac00e | Panagiotis Kanavos | select Task<IList<ObjectInfo>>.Factory.StartNew(_ => |
224 | aa7ac00e | Panagiotis Kanavos | client.ListObjects(accountInfo.UserName, container.Name, since), container.Name)).ToList(); |
225 | 99e6329f | Panagiotis Kanavos | |
226 | 99e6329f | Panagiotis Kanavos | var listShared = Task<IList<ObjectInfo>>.Factory.StartNew(_ => |
227 | 99e6329f | Panagiotis Kanavos | client.ListSharedObjects(since), "shared"); |
228 | aa7ac00e | Panagiotis Kanavos | listObjects.Add(listShared); |
229 | aa7ac00e | Panagiotis Kanavos | var listTasks = await Task.Factory.WhenAll(listObjects.ToArray()); |
230 | aa7ac00e | Panagiotis Kanavos | |
231 | dccd340f | Panagiotis Kanavos | using (ThreadContext.Stacks["SCHEDULE"].Push("Process Results")) |
232 | aa7ac00e | Panagiotis Kanavos | { |
233 | aa7ac00e | Panagiotis Kanavos | var dict = listTasks.ToDictionary(t => t.AsyncState); |
234 | aa7ac00e | Panagiotis Kanavos | |
235 | aa7ac00e | Panagiotis Kanavos | //Get all non-trash objects. Remember, the container name is stored in AsyncState |
236 | 6f03d6e1 | Panagiotis Kanavos | var remoteObjects = (from objectList in listTasks |
237 | aa7ac00e | Panagiotis Kanavos | where (string)objectList.AsyncState != "trash" |
238 | aa7ac00e | Panagiotis Kanavos | from obj in objectList.Result |
239 | 6f03d6e1 | Panagiotis Kanavos | select obj).ToList(); |
240 | 6f03d6e1 | Panagiotis Kanavos | |
241 | 6f03d6e1 | Panagiotis Kanavos | //Get the latest remote object modification date, only if it is after |
242 | 6f03d6e1 | Panagiotis Kanavos | //the original since date |
243 | 6f03d6e1 | Panagiotis Kanavos | nextSince = GetLatestDateAfter(nextSince, remoteObjects); |
244 | aa7ac00e | Panagiotis Kanavos | |
245 | aa7ac00e | Panagiotis Kanavos | var sharedObjects = dict["shared"].Result; |
246 | 6f03d6e1 | Panagiotis Kanavos | nextSince = GetLatestDateBefore(nextSince, sharedObjects); |
247 | aa7ac00e | Panagiotis Kanavos | |
248 | aa7ac00e | Panagiotis Kanavos | //DON'T process trashed files |
249 | aa7ac00e | Panagiotis Kanavos | //If some files are deleted and added again to a folder, they will be deleted |
250 | aa7ac00e | Panagiotis Kanavos | //even though they are new. |
251 | aa7ac00e | Panagiotis Kanavos | //We would have to check file dates and hashes to ensure that a trashed file |
252 | aa7ac00e | Panagiotis Kanavos | //can be deleted safely from the local hard drive. |
253 | aa7ac00e | Panagiotis Kanavos | /* |
254 | aa7ac00e | Panagiotis Kanavos | //Items with the same name, hash may be both in the container and the trash |
255 | aa7ac00e | Panagiotis Kanavos | //Don't delete items that exist in the container |
256 | aa7ac00e | Panagiotis Kanavos | var realTrash = from trash in trashObjects |
257 | aa7ac00e | Panagiotis Kanavos | where |
258 | aa7ac00e | Panagiotis Kanavos | !remoteObjects.Any( |
259 | aa7ac00e | Panagiotis Kanavos | info => info.Name == trash.Name && info.Hash == trash.Hash) |
260 | aa7ac00e | Panagiotis Kanavos | select trash; |
261 | aa7ac00e | Panagiotis Kanavos | ProcessTrashedFiles(accountInfo, realTrash); |
262 | aa7ac00e | Panagiotis Kanavos | */ |
263 | aa7ac00e | Panagiotis Kanavos | |
264 | aa7ac00e | Panagiotis Kanavos | var cleanRemotes = (from info in remoteObjects.Union(sharedObjects) |
265 | 92f18b56 | Panagiotis Kanavos | let name = info.Name??"" |
266 | aa7ac00e | Panagiotis Kanavos | where !name.EndsWith(".ignore", StringComparison.InvariantCultureIgnoreCase) && |
267 | aa7ac00e | Panagiotis Kanavos | !name.StartsWith(FolderConstants.CacheFolder + "/", |
268 | aa7ac00e | Panagiotis Kanavos | StringComparison.InvariantCultureIgnoreCase) |
269 | aa7ac00e | Panagiotis Kanavos | select info).ToList(); |
270 | aa7ac00e | Panagiotis Kanavos | |
271 | aa7ac00e | Panagiotis Kanavos | var differencer = _differencer.PostSnapshot(accountInfo, cleanRemotes); |
272 | aa7ac00e | Panagiotis Kanavos | |
273 | ec1a1baf | Panagiotis Kanavos | ProcessDeletedFiles(accountInfo, differencer.Deleted.FilterDirectlyBelow(SelectiveUris)); |
274 | 759bd3c4 | Panagiotis Kanavos | |
275 | 759bd3c4 | Panagiotis Kanavos | // @@@ NEED To add previous state here as well, To compare with previous hash |
276 | 759bd3c4 | Panagiotis Kanavos | |
277 | 759bd3c4 | Panagiotis Kanavos | |
278 | aa7ac00e | Panagiotis Kanavos | |
279 | aa7ac00e | Panagiotis Kanavos | //Create a list of actions from the remote files |
280 | b666b39a | Panagiotis Kanavos | var allActions = MovesToActions(accountInfo,differencer.Moved.FilterDirectlyBelow(SelectiveUris)) |
281 | b666b39a | Panagiotis Kanavos | .Union( |
282 | b666b39a | Panagiotis Kanavos | ChangesToActions(accountInfo, differencer.Changed.FilterDirectlyBelow(SelectiveUris))) |
283 | aa7ac00e | Panagiotis Kanavos | .Union( |
284 | fec5da06 | Panagiotis Kanavos | CreatesToActions(accountInfo, differencer.Created.FilterDirectlyBelow(SelectiveUris))); |
285 | aa7ac00e | Panagiotis Kanavos | |
286 | aa7ac00e | Panagiotis Kanavos | //And remove those that are already being processed by the agent |
287 | aa7ac00e | Panagiotis Kanavos | var distinctActions = allActions |
288 | dccd340f | Panagiotis Kanavos | .Except(NetworkAgent.GetEnumerable(), new LocalFileComparer()) |
289 | aa7ac00e | Panagiotis Kanavos | .ToList(); |
290 | aa7ac00e | Panagiotis Kanavos | |
291 | aa7ac00e | Panagiotis Kanavos | //Queue all the actions |
292 | aa7ac00e | Panagiotis Kanavos | foreach (var message in distinctActions) |
293 | aa7ac00e | Panagiotis Kanavos | { |
294 | aa7ac00e | Panagiotis Kanavos | NetworkAgent.Post(message); |
295 | aa7ac00e | Panagiotis Kanavos | } |
296 | aa7ac00e | Panagiotis Kanavos | |
297 | aa7ac00e | Panagiotis Kanavos | Log.Info("[LISTENER] End Processing"); |
298 | aa7ac00e | Panagiotis Kanavos | } |
299 | aa7ac00e | Panagiotis Kanavos | } |
300 | aa7ac00e | Panagiotis Kanavos | catch (Exception ex) |
301 | aa7ac00e | Panagiotis Kanavos | { |
302 | aa7ac00e | Panagiotis Kanavos | Log.ErrorFormat("[FAIL] ListObjects for{0} in ProcessRemoteFiles with {1}", accountInfo.UserName, ex); |
303 | 6f03d6e1 | Panagiotis Kanavos | return nextSince; |
304 | aa7ac00e | Panagiotis Kanavos | } |
305 | aa7ac00e | Panagiotis Kanavos | |
306 | aa7ac00e | Panagiotis Kanavos | Log.Info("[LISTENER] Finished"); |
307 | 6f03d6e1 | Panagiotis Kanavos | return nextSince; |
308 | aa7ac00e | Panagiotis Kanavos | } |
309 | aa7ac00e | Panagiotis Kanavos | } |
310 | aa7ac00e | Panagiotis Kanavos | |
311 | 6f03d6e1 | Panagiotis Kanavos | /// <summary> |
312 | 6f03d6e1 | Panagiotis Kanavos | /// Returns the latest LastModified date from the list of objects, but only if it is before |
313 | 6f03d6e1 | Panagiotis Kanavos | /// than the threshold value |
314 | 6f03d6e1 | Panagiotis Kanavos | /// </summary> |
315 | 6f03d6e1 | Panagiotis Kanavos | /// <param name="threshold"></param> |
316 | 6f03d6e1 | Panagiotis Kanavos | /// <param name="cloudObjects"></param> |
317 | 6f03d6e1 | Panagiotis Kanavos | /// <returns></returns> |
318 | 6f03d6e1 | Panagiotis Kanavos | private static DateTime? GetLatestDateBefore(DateTime? threshold, IList<ObjectInfo> cloudObjects) |
319 | 6f03d6e1 | Panagiotis Kanavos | { |
320 | 6f03d6e1 | Panagiotis Kanavos | DateTime? maxDate = null; |
321 | 6f03d6e1 | Panagiotis Kanavos | if (cloudObjects!=null && cloudObjects.Count > 0) |
322 | 6f03d6e1 | Panagiotis Kanavos | maxDate = cloudObjects.Max(obj => obj.Last_Modified); |
323 | 6f03d6e1 | Panagiotis Kanavos | if (maxDate == null || maxDate == DateTime.MinValue) |
324 | 6f03d6e1 | Panagiotis Kanavos | return threshold; |
325 | 6f03d6e1 | Panagiotis Kanavos | if (threshold == null || threshold == DateTime.MinValue || threshold > maxDate) |
326 | 6f03d6e1 | Panagiotis Kanavos | return maxDate; |
327 | 6f03d6e1 | Panagiotis Kanavos | return threshold; |
328 | 6f03d6e1 | Panagiotis Kanavos | } |
329 | 6f03d6e1 | Panagiotis Kanavos | |
330 | 6f03d6e1 | Panagiotis Kanavos | /// <summary> |
331 | 6f03d6e1 | Panagiotis Kanavos | /// Returns the latest LastModified date from the list of objects, but only if it is after |
332 | 6f03d6e1 | Panagiotis Kanavos | /// the threshold value |
333 | 6f03d6e1 | Panagiotis Kanavos | /// </summary> |
334 | 6f03d6e1 | Panagiotis Kanavos | /// <param name="threshold"></param> |
335 | 6f03d6e1 | Panagiotis Kanavos | /// <param name="cloudObjects"></param> |
336 | 6f03d6e1 | Panagiotis Kanavos | /// <returns></returns> |
337 | 6f03d6e1 | Panagiotis Kanavos | private static DateTime? GetLatestDateAfter(DateTime? threshold, IList<ObjectInfo> cloudObjects) |
338 | 6f03d6e1 | Panagiotis Kanavos | { |
339 | 6f03d6e1 | Panagiotis Kanavos | DateTime? maxDate = null; |
340 | 6f03d6e1 | Panagiotis Kanavos | if (cloudObjects!=null && cloudObjects.Count > 0) |
341 | 6f03d6e1 | Panagiotis Kanavos | maxDate = cloudObjects.Max(obj => obj.Last_Modified); |
342 | 6f03d6e1 | Panagiotis Kanavos | if (maxDate == null || maxDate == DateTime.MinValue) |
343 | 6f03d6e1 | Panagiotis Kanavos | return threshold; |
344 | 6f03d6e1 | Panagiotis Kanavos | if (threshold == null || threshold == DateTime.MinValue || threshold < maxDate) |
345 | 6f03d6e1 | Panagiotis Kanavos | return maxDate; |
346 | 6f03d6e1 | Panagiotis Kanavos | return threshold; |
347 | 6f03d6e1 | Panagiotis Kanavos | } |
348 | 6f03d6e1 | Panagiotis Kanavos | |
349 | ec1a1baf | Panagiotis Kanavos | readonly AccountsDifferencer _differencer = new AccountsDifferencer(); |
350 | 759bd3c4 | Panagiotis Kanavos | private List<Uri> _selectiveUris=new List<Uri>(); |
351 | aa7ac00e | Panagiotis Kanavos | |
352 | aa7ac00e | Panagiotis Kanavos | /// <summary> |
353 | aa7ac00e | Panagiotis Kanavos | /// Deletes local files that are not found in the list of cloud files |
354 | aa7ac00e | Panagiotis Kanavos | /// </summary> |
355 | aa7ac00e | Panagiotis Kanavos | /// <param name="accountInfo"></param> |
356 | aa7ac00e | Panagiotis Kanavos | /// <param name="cloudFiles"></param> |
357 | ec1a1baf | Panagiotis Kanavos | private void ProcessDeletedFiles(AccountInfo accountInfo, IEnumerable<ObjectInfo> cloudFiles) |
358 | aa7ac00e | Panagiotis Kanavos | { |
359 | aa7ac00e | Panagiotis Kanavos | if (accountInfo == null) |
360 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentNullException("accountInfo"); |
361 | aa7ac00e | Panagiotis Kanavos | if (String.IsNullOrWhiteSpace(accountInfo.AccountPath)) |
362 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentException("The AccountInfo.AccountPath is empty", "accountInfo"); |
363 | aa7ac00e | Panagiotis Kanavos | if (cloudFiles == null) |
364 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentNullException("cloudFiles"); |
365 | aa7ac00e | Panagiotis Kanavos | Contract.EndContractBlock(); |
366 | aa7ac00e | Panagiotis Kanavos | |
367 | aa7ac00e | Panagiotis Kanavos | //On the first run |
368 | aa7ac00e | Panagiotis Kanavos | if (_firstPoll) |
369 | aa7ac00e | Panagiotis Kanavos | { |
370 | aa7ac00e | Panagiotis Kanavos | //Only consider files that are not being modified, ie they are in the Unchanged state |
371 | aa7ac00e | Panagiotis Kanavos | var deleteCandidates = FileState.Queryable.Where(state => |
372 | aa7ac00e | Panagiotis Kanavos | state.FilePath.StartsWith(accountInfo.AccountPath) |
373 | aa7ac00e | Panagiotis Kanavos | && state.FileStatus == FileStatus.Unchanged).ToList(); |
374 | aa7ac00e | Panagiotis Kanavos | |
375 | aa7ac00e | Panagiotis Kanavos | |
376 | aa7ac00e | Panagiotis Kanavos | //TODO: filesToDelete must take into account the Others container |
377 | aa7ac00e | Panagiotis Kanavos | var filesToDelete = (from deleteCandidate in deleteCandidates |
378 | aa7ac00e | Panagiotis Kanavos | let localFile = FileInfoExtensions.FromPath(deleteCandidate.FilePath) |
379 | aa7ac00e | Panagiotis Kanavos | let relativeFilePath = localFile.AsRelativeTo(accountInfo.AccountPath) |
380 | aa7ac00e | Panagiotis Kanavos | where |
381 | aa7ac00e | Panagiotis Kanavos | !cloudFiles.Any(r => r.RelativeUrlToFilePath(accountInfo.UserName) == relativeFilePath) |
382 | aa7ac00e | Panagiotis Kanavos | select localFile).ToList(); |
383 | aa7ac00e | Panagiotis Kanavos | |
384 | aa7ac00e | Panagiotis Kanavos | |
385 | aa7ac00e | Panagiotis Kanavos | |
386 | aa7ac00e | Panagiotis Kanavos | //Set the status of missing files to Conflict |
387 | aa7ac00e | Panagiotis Kanavos | foreach (var item in filesToDelete) |
388 | aa7ac00e | Panagiotis Kanavos | { |
389 | aa7ac00e | Panagiotis Kanavos | //Try to acquire a gate on the file, to take into account files that have been dequeued |
390 | aa7ac00e | Panagiotis Kanavos | //and are being processed |
391 | aa7ac00e | Panagiotis Kanavos | using (var gate = NetworkGate.Acquire(item.FullName, NetworkOperation.Deleting)) |
392 | aa7ac00e | Panagiotis Kanavos | { |
393 | aa7ac00e | Panagiotis Kanavos | if (gate.Failed) |
394 | aa7ac00e | Panagiotis Kanavos | continue; |
395 | aa7ac00e | Panagiotis Kanavos | StatusKeeper.SetFileState(item.FullName, FileStatus.Conflict, FileOverlayStatus.Deleted); |
396 | aa7ac00e | Panagiotis Kanavos | } |
397 | aa7ac00e | Panagiotis Kanavos | } |
398 | aa7ac00e | Panagiotis Kanavos | UpdateStatus(PithosStatus.HasConflicts); |
399 | aa7ac00e | Panagiotis Kanavos | StatusNotification.NotifyConflicts(filesToDelete, String.Format("{0} local files are missing from Pithos, possibly because they were deleted", filesToDelete.Count)); |
400 | aa7ac00e | Panagiotis Kanavos | StatusNotification.NotifyForFiles(filesToDelete, String.Format("{0} files were deleted", filesToDelete.Count), TraceLevel.Info); |
401 | aa7ac00e | Panagiotis Kanavos | } |
402 | aa7ac00e | Panagiotis Kanavos | else |
403 | aa7ac00e | Panagiotis Kanavos | { |
404 | aa7ac00e | Panagiotis Kanavos | var deletedFiles = new List<FileSystemInfo>(); |
405 | aa7ac00e | Panagiotis Kanavos | foreach (var objectInfo in cloudFiles) |
406 | aa7ac00e | Panagiotis Kanavos | { |
407 | 6bcdd8e2 | Panagiotis Kanavos | Log.DebugFormat("Handle deleted [{0}]",objectInfo.Uri); |
408 | aa7ac00e | Panagiotis Kanavos | var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName); |
409 | fbbbe99b | Panagiotis Kanavos | var item = FileAgent.GetFileAgent(accountInfo).GetFileSystemInfo(relativePath); |
410 | 6bcdd8e2 | Panagiotis Kanavos | Log.DebugFormat("Will delete [{0}] for [{1}]", item.FullName,objectInfo.Uri); |
411 | aa7ac00e | Panagiotis Kanavos | if (item.Exists) |
412 | aa7ac00e | Panagiotis Kanavos | { |
413 | aa7ac00e | Panagiotis Kanavos | if ((item.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) |
414 | aa7ac00e | Panagiotis Kanavos | { |
415 | aa7ac00e | Panagiotis Kanavos | item.Attributes = item.Attributes & ~FileAttributes.ReadOnly; |
416 | aa7ac00e | Panagiotis Kanavos | |
417 | aa7ac00e | Panagiotis Kanavos | } |
418 | 6bcdd8e2 | Panagiotis Kanavos | var directory = item as DirectoryInfo; |
419 | 6bcdd8e2 | Panagiotis Kanavos | if (directory!=null) |
420 | 6bcdd8e2 | Panagiotis Kanavos | directory.Delete(true); |
421 | 6bcdd8e2 | Panagiotis Kanavos | else |
422 | 6bcdd8e2 | Panagiotis Kanavos | item.Delete(); |
423 | 6bcdd8e2 | Panagiotis Kanavos | Log.DebugFormat("Deleted [{0}] for [{1}]", item.FullName, objectInfo.Uri); |
424 | aa7ac00e | Panagiotis Kanavos | DateTime lastDate; |
425 | aa7ac00e | Panagiotis Kanavos | _lastSeen.TryRemove(item.FullName, out lastDate); |
426 | aa7ac00e | Panagiotis Kanavos | deletedFiles.Add(item); |
427 | aa7ac00e | Panagiotis Kanavos | } |
428 | aa7ac00e | Panagiotis Kanavos | StatusKeeper.SetFileState(item.FullName, FileStatus.Deleted, FileOverlayStatus.Deleted); |
429 | aa7ac00e | Panagiotis Kanavos | } |
430 | 6bcdd8e2 | Panagiotis Kanavos | Log.InfoFormat("[{0}] files were deleted",deletedFiles.Count); |
431 | aa7ac00e | Panagiotis Kanavos | StatusNotification.NotifyForFiles(deletedFiles, String.Format("{0} files were deleted", deletedFiles.Count), TraceLevel.Info); |
432 | aa7ac00e | Panagiotis Kanavos | } |
433 | aa7ac00e | Panagiotis Kanavos | |
434 | aa7ac00e | Panagiotis Kanavos | } |
435 | aa7ac00e | Panagiotis Kanavos | |
436 | b666b39a | Panagiotis Kanavos | /// <summary> |
437 | b666b39a | Panagiotis Kanavos | /// Creates a Sync action for each changed server file |
438 | b666b39a | Panagiotis Kanavos | /// </summary> |
439 | b666b39a | Panagiotis Kanavos | /// <param name="accountInfo"></param> |
440 | b666b39a | Panagiotis Kanavos | /// <param name="changes"></param> |
441 | b666b39a | Panagiotis Kanavos | /// <returns></returns> |
442 | aa7ac00e | Panagiotis Kanavos | private IEnumerable<CloudAction> ChangesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> changes) |
443 | aa7ac00e | Panagiotis Kanavos | { |
444 | aa7ac00e | Panagiotis Kanavos | if (changes == null) |
445 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentNullException(); |
446 | aa7ac00e | Panagiotis Kanavos | Contract.EndContractBlock(); |
447 | fbbbe99b | Panagiotis Kanavos | var fileAgent = FileAgent.GetFileAgent(accountInfo); |
448 | aa7ac00e | Panagiotis Kanavos | |
449 | aa7ac00e | Panagiotis Kanavos | //In order to avoid multiple iterations over the files, we iterate only once |
450 | aa7ac00e | Panagiotis Kanavos | //over the remote files |
451 | aa7ac00e | Panagiotis Kanavos | foreach (var objectInfo in changes) |
452 | aa7ac00e | Panagiotis Kanavos | { |
453 | aa7ac00e | Panagiotis Kanavos | var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName); |
454 | b666b39a | Panagiotis Kanavos | //If a directory object already exists, we may need to sync it |
455 | aa7ac00e | Panagiotis Kanavos | if (fileAgent.Exists(relativePath)) |
456 | aa7ac00e | Panagiotis Kanavos | { |
457 | aa7ac00e | Panagiotis Kanavos | var localFile = fileAgent.GetFileSystemInfo(relativePath); |
458 | b666b39a | Panagiotis Kanavos | //We don't need to sync directories |
459 | aa7ac00e | Panagiotis Kanavos | if (objectInfo.Content_Type == @"application/directory" && localFile is DirectoryInfo) |
460 | aa7ac00e | Panagiotis Kanavos | continue; |
461 | aa7ac00e | Panagiotis Kanavos | using (new SessionScope(FlushAction.Never)) |
462 | aa7ac00e | Panagiotis Kanavos | { |
463 | aa7ac00e | Panagiotis Kanavos | var state = StatusKeeper.GetStateByFilePath(localFile.FullName); |
464 | aa7ac00e | Panagiotis Kanavos | _lastSeen[localFile.FullName] = DateTime.Now; |
465 | aa7ac00e | Panagiotis Kanavos | //Common files should be checked on a per-case basis to detect differences, which is newer |
466 | aa7ac00e | Panagiotis Kanavos | |
467 | aa7ac00e | Panagiotis Kanavos | yield return new CloudAction(accountInfo, CloudActionType.MustSynch, |
468 | aa7ac00e | Panagiotis Kanavos | localFile, objectInfo, state, accountInfo.BlockSize, |
469 | aa7ac00e | Panagiotis Kanavos | accountInfo.BlockHash); |
470 | aa7ac00e | Panagiotis Kanavos | } |
471 | aa7ac00e | Panagiotis Kanavos | } |
472 | aa7ac00e | Panagiotis Kanavos | else |
473 | aa7ac00e | Panagiotis Kanavos | { |
474 | aa7ac00e | Panagiotis Kanavos | //Remote files should be downloaded |
475 | aa7ac00e | Panagiotis Kanavos | yield return new CloudDownloadAction(accountInfo, objectInfo); |
476 | aa7ac00e | Panagiotis Kanavos | } |
477 | aa7ac00e | Panagiotis Kanavos | } |
478 | aa7ac00e | Panagiotis Kanavos | } |
479 | aa7ac00e | Panagiotis Kanavos | |
480 | b666b39a | Panagiotis Kanavos | /// <summary> |
481 | b666b39a | Panagiotis Kanavos | /// Creates a Local Move action for each moved server file |
482 | b666b39a | Panagiotis Kanavos | /// </summary> |
483 | b666b39a | Panagiotis Kanavos | /// <param name="accountInfo"></param> |
484 | b666b39a | Panagiotis Kanavos | /// <param name="moves"></param> |
485 | b666b39a | Panagiotis Kanavos | /// <returns></returns> |
486 | b666b39a | Panagiotis Kanavos | private IEnumerable<CloudAction> MovesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> moves) |
487 | b666b39a | Panagiotis Kanavos | { |
488 | b666b39a | Panagiotis Kanavos | if (moves == null) |
489 | b666b39a | Panagiotis Kanavos | throw new ArgumentNullException(); |
490 | b666b39a | Panagiotis Kanavos | Contract.EndContractBlock(); |
491 | b666b39a | Panagiotis Kanavos | var fileAgent = FileAgent.GetFileAgent(accountInfo); |
492 | b666b39a | Panagiotis Kanavos | |
493 | b666b39a | Panagiotis Kanavos | //In order to avoid multiple iterations over the files, we iterate only once |
494 | b666b39a | Panagiotis Kanavos | //over the remote files |
495 | b666b39a | Panagiotis Kanavos | foreach (var objectInfo in moves) |
496 | b666b39a | Panagiotis Kanavos | { |
497 | b666b39a | Panagiotis Kanavos | var previousRelativepath = objectInfo.Previous.RelativeUrlToFilePath(accountInfo.UserName); |
498 | b666b39a | Panagiotis Kanavos | //If the previous file already exists, we can execute a Move operation |
499 | b666b39a | Panagiotis Kanavos | if (fileAgent.Exists(previousRelativepath)) |
500 | b666b39a | Panagiotis Kanavos | { |
501 | b666b39a | Panagiotis Kanavos | var previousFile = fileAgent.GetFileSystemInfo(previousRelativepath); |
502 | b666b39a | Panagiotis Kanavos | using (new SessionScope(FlushAction.Never)) |
503 | b666b39a | Panagiotis Kanavos | { |
504 | b666b39a | Panagiotis Kanavos | var state = StatusKeeper.GetStateByFilePath(previousFile.FullName); |
505 | b666b39a | Panagiotis Kanavos | _lastSeen[previousFile.FullName] = DateTime.Now; |
506 | b666b39a | Panagiotis Kanavos | |
507 | b666b39a | Panagiotis Kanavos | //For each moved object we need to move both the local file and update |
508 | b666b39a | Panagiotis Kanavos | yield return new CloudAction(accountInfo, CloudActionType.RenameLocal, |
509 | b666b39a | Panagiotis Kanavos | previousFile, objectInfo, state, accountInfo.BlockSize, |
510 | b666b39a | Panagiotis Kanavos | accountInfo.BlockHash); |
511 | b666b39a | Panagiotis Kanavos | //For modified files, we need to download the changes as well |
512 | b666b39a | Panagiotis Kanavos | if (objectInfo.Hash!=objectInfo.PreviousHash) |
513 | b666b39a | Panagiotis Kanavos | yield return new CloudDownloadAction(accountInfo,objectInfo); |
514 | b666b39a | Panagiotis Kanavos | } |
515 | b666b39a | Panagiotis Kanavos | } |
516 | b666b39a | Panagiotis Kanavos | //If the previous file does not exist, we need to download it in the new location |
517 | b666b39a | Panagiotis Kanavos | else |
518 | b666b39a | Panagiotis Kanavos | { |
519 | b666b39a | Panagiotis Kanavos | //Remote files should be downloaded |
520 | b666b39a | Panagiotis Kanavos | yield return new CloudDownloadAction(accountInfo, objectInfo); |
521 | b666b39a | Panagiotis Kanavos | } |
522 | b666b39a | Panagiotis Kanavos | } |
523 | b666b39a | Panagiotis Kanavos | } |
524 | b666b39a | Panagiotis Kanavos | |
525 | b666b39a | Panagiotis Kanavos | |
526 | b666b39a | Panagiotis Kanavos | /// <summary> |
527 | b666b39a | Panagiotis Kanavos | /// Creates a download action for each new server file |
528 | b666b39a | Panagiotis Kanavos | /// </summary> |
529 | b666b39a | Panagiotis Kanavos | /// <param name="accountInfo"></param> |
530 | b666b39a | Panagiotis Kanavos | /// <param name="creates"></param> |
531 | b666b39a | Panagiotis Kanavos | /// <returns></returns> |
532 | aa7ac00e | Panagiotis Kanavos | private IEnumerable<CloudAction> CreatesToActions(AccountInfo accountInfo, IEnumerable<ObjectInfo> creates) |
533 | aa7ac00e | Panagiotis Kanavos | { |
534 | aa7ac00e | Panagiotis Kanavos | if (creates == null) |
535 | aa7ac00e | Panagiotis Kanavos | throw new ArgumentNullException(); |
536 | aa7ac00e | Panagiotis Kanavos | Contract.EndContractBlock(); |
537 | fbbbe99b | Panagiotis Kanavos | var fileAgent = FileAgent.GetFileAgent(accountInfo); |
538 | aa7ac00e | Panagiotis Kanavos | |
539 | aa7ac00e | Panagiotis Kanavos | //In order to avoid multiple iterations over the files, we iterate only once |
540 | aa7ac00e | Panagiotis Kanavos | //over the remote files |
541 | aa7ac00e | Panagiotis Kanavos | foreach (var objectInfo in creates) |
542 | aa7ac00e | Panagiotis Kanavos | { |
543 | dccd340f | Panagiotis Kanavos | if (Log.IsDebugEnabled) |
544 | dccd340f | Panagiotis Kanavos | Log.DebugFormat("[NEW INFO] {0}",objectInfo.Uri); |
545 | dccd340f | Panagiotis Kanavos | |
546 | aa7ac00e | Panagiotis Kanavos | var relativePath = objectInfo.RelativeUrlToFilePath(accountInfo.UserName); |
547 | b666b39a | Panagiotis Kanavos | //If the object already exists, we probably have a conflict |
548 | aa7ac00e | Panagiotis Kanavos | if (fileAgent.Exists(relativePath)) |
549 | aa7ac00e | Panagiotis Kanavos | { |
550 | dccd340f | Panagiotis Kanavos | Log.DebugFormat("[SKIP EXISTING] {0}", objectInfo.Uri); |
551 | aa7ac00e | Panagiotis Kanavos | //If a directory object already exists, we don't need to perform any other action |
552 | aa7ac00e | Panagiotis Kanavos | var localFile = fileAgent.GetFileSystemInfo(relativePath); |
553 | aa7ac00e | Panagiotis Kanavos | StatusKeeper.SetFileState(localFile.FullName, FileStatus.Conflict, FileOverlayStatus.Conflict); |
554 | aa7ac00e | Panagiotis Kanavos | } |
555 | aa7ac00e | Panagiotis Kanavos | else |
556 | aa7ac00e | Panagiotis Kanavos | { |
557 | aa7ac00e | Panagiotis Kanavos | //Remote files should be downloaded |
558 | aa7ac00e | Panagiotis Kanavos | yield return new CloudDownloadAction(accountInfo, objectInfo); |
559 | aa7ac00e | Panagiotis Kanavos | } |
560 | aa7ac00e | Panagiotis Kanavos | } |
561 | aa7ac00e | Panagiotis Kanavos | } |
562 | aa7ac00e | Panagiotis Kanavos | |
563 | fec5da06 | Panagiotis Kanavos | /// <summary> |
564 | fec5da06 | Panagiotis Kanavos | /// Notify the UI to update the visual status |
565 | fec5da06 | Panagiotis Kanavos | /// </summary> |
566 | fec5da06 | Panagiotis Kanavos | /// <param name="status"></param> |
567 | aa7ac00e | Panagiotis Kanavos | private void UpdateStatus(PithosStatus status) |
568 | aa7ac00e | Panagiotis Kanavos | { |
569 | fec5da06 | Panagiotis Kanavos | try |
570 | fec5da06 | Panagiotis Kanavos | { |
571 | 174bbb6e | Panagiotis Kanavos | StatusNotification.SetPithosStatus(status); |
572 | 174bbb6e | Panagiotis Kanavos | //StatusNotification.Notify(new Notification()); |
573 | fec5da06 | Panagiotis Kanavos | } |
574 | fec5da06 | Panagiotis Kanavos | catch (Exception exc) |
575 | fec5da06 | Panagiotis Kanavos | { |
576 | fec5da06 | Panagiotis Kanavos | //Failure is not critical, just log it |
577 | fec5da06 | Panagiotis Kanavos | Log.Warn("Error while updating status", exc); |
578 | fec5da06 | Panagiotis Kanavos | } |
579 | aa7ac00e | Panagiotis Kanavos | } |
580 | aa7ac00e | Panagiotis Kanavos | |
581 | aa7ac00e | Panagiotis Kanavos | private static void CreateContainerFolders(AccountInfo accountInfo, IEnumerable<ContainerInfo> containers) |
582 | aa7ac00e | Panagiotis Kanavos | { |
583 | aa7ac00e | Panagiotis Kanavos | var containerPaths = from container in containers |
584 | aa7ac00e | Panagiotis Kanavos | let containerPath = Path.Combine(accountInfo.AccountPath, container.Name) |
585 | aa7ac00e | Panagiotis Kanavos | where container.Name != FolderConstants.TrashContainer && !Directory.Exists(containerPath) |
586 | aa7ac00e | Panagiotis Kanavos | select containerPath; |
587 | aa7ac00e | Panagiotis Kanavos | |
588 | aa7ac00e | Panagiotis Kanavos | foreach (var path in containerPaths) |
589 | aa7ac00e | Panagiotis Kanavos | { |
590 | aa7ac00e | Panagiotis Kanavos | Directory.CreateDirectory(path); |
591 | aa7ac00e | Panagiotis Kanavos | } |
592 | aa7ac00e | Panagiotis Kanavos | } |
593 | 759bd3c4 | Panagiotis Kanavos | |
594 | fec5da06 | Panagiotis Kanavos | public void SetSyncUris(Uri[] uris) |
595 | fec5da06 | Panagiotis Kanavos | { |
596 | fec5da06 | Panagiotis Kanavos | SelectiveUris=uris.ToList(); |
597 | 759bd3c4 | Panagiotis Kanavos | } |
598 | 759bd3c4 | Panagiotis Kanavos | |
599 | 759bd3c4 | Panagiotis Kanavos | protected List<Uri> SelectiveUris |
600 | 759bd3c4 | Panagiotis Kanavos | { |
601 | 759bd3c4 | Panagiotis Kanavos | get { return _selectiveUris;} |
602 | 759bd3c4 | Panagiotis Kanavos | set { _selectiveUris = value; } |
603 | 759bd3c4 | Panagiotis Kanavos | } |
604 | fec5da06 | Panagiotis Kanavos | |
605 | fec5da06 | Panagiotis Kanavos | public void AddAccount(AccountInfo accountInfo) |
606 | fec5da06 | Panagiotis Kanavos | { |
607 | ec1a1baf | Panagiotis Kanavos | //Avoid adding a duplicate accountInfo |
608 | ec1a1baf | Panagiotis Kanavos | _accounts.TryAdd(accountInfo.UserName, accountInfo); |
609 | ec1a1baf | Panagiotis Kanavos | } |
610 | ec1a1baf | Panagiotis Kanavos | |
611 | ec1a1baf | Panagiotis Kanavos | public void RemoveAccount(AccountInfo accountInfo) |
612 | ec1a1baf | Panagiotis Kanavos | { |
613 | ec1a1baf | Panagiotis Kanavos | AccountInfo account; |
614 | ec1a1baf | Panagiotis Kanavos | _accounts.TryRemove(accountInfo.UserName,out account); |
615 | dccd340f | Panagiotis Kanavos | SnapshotDifferencer differencer; |
616 | dccd340f | Panagiotis Kanavos | _differencer.Differencers.TryRemove(accountInfo.UserName, out differencer); |
617 | fec5da06 | Panagiotis Kanavos | } |
618 | aa7ac00e | Panagiotis Kanavos | } |
619 | aa7ac00e | Panagiotis Kanavos | } |