root / trunk / Pithos.Core / Agents / DeleteAgent.cs @ dccd340f
History | View | Annotate | Download (9.8 kB)
1 | 255f5f86 | Panagiotis Kanavos | #region |
---|---|---|---|
2 | 255f5f86 | Panagiotis Kanavos | /* ----------------------------------------------------------------------- |
3 | 255f5f86 | Panagiotis Kanavos | * <copyright file="DeleteAgent.cs" company="GRNet"> |
4 | 255f5f86 | Panagiotis Kanavos | * |
5 | 255f5f86 | Panagiotis Kanavos | * Copyright 2011-2012 GRNET S.A. All rights reserved. |
6 | 255f5f86 | Panagiotis Kanavos | * |
7 | 255f5f86 | Panagiotis Kanavos | * Redistribution and use in source and binary forms, with or |
8 | 255f5f86 | Panagiotis Kanavos | * without modification, are permitted provided that the following |
9 | 255f5f86 | Panagiotis Kanavos | * conditions are met: |
10 | 255f5f86 | Panagiotis Kanavos | * |
11 | 255f5f86 | Panagiotis Kanavos | * 1. Redistributions of source code must retain the above |
12 | 255f5f86 | Panagiotis Kanavos | * copyright notice, this list of conditions and the following |
13 | 255f5f86 | Panagiotis Kanavos | * disclaimer. |
14 | 255f5f86 | Panagiotis Kanavos | * |
15 | 255f5f86 | Panagiotis Kanavos | * 2. Redistributions in binary form must reproduce the above |
16 | 255f5f86 | Panagiotis Kanavos | * copyright notice, this list of conditions and the following |
17 | 255f5f86 | Panagiotis Kanavos | * disclaimer in the documentation and/or other materials |
18 | 255f5f86 | Panagiotis Kanavos | * provided with the distribution. |
19 | 255f5f86 | Panagiotis Kanavos | * |
20 | 255f5f86 | Panagiotis Kanavos | * |
21 | 255f5f86 | Panagiotis Kanavos | * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
22 | 255f5f86 | Panagiotis Kanavos | * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
23 | 255f5f86 | Panagiotis Kanavos | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
24 | 255f5f86 | Panagiotis Kanavos | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
25 | 255f5f86 | Panagiotis Kanavos | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
26 | 255f5f86 | Panagiotis Kanavos | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
27 | 255f5f86 | Panagiotis Kanavos | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
28 | 255f5f86 | Panagiotis Kanavos | * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
29 | 255f5f86 | Panagiotis Kanavos | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
30 | 255f5f86 | Panagiotis Kanavos | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
31 | 255f5f86 | Panagiotis Kanavos | * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
32 | 255f5f86 | Panagiotis Kanavos | * POSSIBILITY OF SUCH DAMAGE. |
33 | 255f5f86 | Panagiotis Kanavos | * |
34 | 255f5f86 | Panagiotis Kanavos | * The views and conclusions contained in the software and |
35 | 255f5f86 | Panagiotis Kanavos | * documentation are those of the authors and should not be |
36 | 255f5f86 | Panagiotis Kanavos | * interpreted as representing official policies, either expressed |
37 | 255f5f86 | Panagiotis Kanavos | * or implied, of GRNET S.A. |
38 | 255f5f86 | Panagiotis Kanavos | * </copyright> |
39 | 255f5f86 | Panagiotis Kanavos | * ----------------------------------------------------------------------- |
40 | 255f5f86 | Panagiotis Kanavos | */ |
41 | 255f5f86 | Panagiotis Kanavos | #endregion |
42 | 3742088d | Panagiotis Kanavos | using System.Collections.Concurrent; |
43 | 3742088d | Panagiotis Kanavos | using System.ComponentModel.Composition; |
44 | 3742088d | Panagiotis Kanavos | using System.Diagnostics.Contracts; |
45 | 3742088d | Panagiotis Kanavos | using System.IO; |
46 | 3742088d | Panagiotis Kanavos | using System.Net; |
47 | db8a9589 | Panagiotis Kanavos | using System.Reflection; |
48 | 3742088d | Panagiotis Kanavos | using System.Threading.Tasks.Dataflow; |
49 | 3742088d | Panagiotis Kanavos | using Pithos.Interfaces; |
50 | 3742088d | Panagiotis Kanavos | using Pithos.Network; |
51 | 3742088d | Panagiotis Kanavos | using System; |
52 | 3742088d | Panagiotis Kanavos | using log4net; |
53 | 3742088d | Panagiotis Kanavos | |
54 | 3742088d | Panagiotis Kanavos | namespace Pithos.Core.Agents |
55 | 3742088d | Panagiotis Kanavos | { |
56 | 3742088d | Panagiotis Kanavos | |
57 | 3742088d | Panagiotis Kanavos | /// <summary> |
58 | 3742088d | Panagiotis Kanavos | /// The Delete Agent is used to delete files from the Pithos server with high priority, |
59 | 38ac43a6 | Panagiotis Kanavos | /// blocking the network agent through the ProceedEvent until all pending deletions complete |
60 | 3742088d | Panagiotis Kanavos | /// </summary> |
61 | 3742088d | Panagiotis Kanavos | [Export] |
62 | 3742088d | Panagiotis Kanavos | public class DeleteAgent |
63 | 3742088d | Panagiotis Kanavos | { |
64 | db8a9589 | Panagiotis Kanavos | private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); |
65 | 3742088d | Panagiotis Kanavos | |
66 | 3742088d | Panagiotis Kanavos | [Import] |
67 | 3742088d | Panagiotis Kanavos | public IStatusKeeper StatusKeeper { get; set; } |
68 | 3742088d | Panagiotis Kanavos | |
69 | 3742088d | Panagiotis Kanavos | //A separate agent is used to execute delete actions immediatelly; |
70 | 3742088d | Panagiotis Kanavos | private readonly ActionBlock<CloudDeleteAction> _deleteAgent; |
71 | 3742088d | Panagiotis Kanavos | |
72 | 38ac43a6 | Panagiotis Kanavos | //The Proceed event signals the the network agent to proceed with processing. |
73 | 38ac43a6 | Panagiotis Kanavos | //Essentially this event pauses the network agent to give priority to the deletion agent |
74 | 3742088d | Panagiotis Kanavos | //Initially the event is signalled because we don't need to pause |
75 | 38ac43a6 | Panagiotis Kanavos | private readonly AsyncManualResetEvent _proceedEvent = new AsyncManualResetEvent(true); |
76 | 3742088d | Panagiotis Kanavos | |
77 | 38ac43a6 | Panagiotis Kanavos | public AsyncManualResetEvent ProceedEvent |
78 | 3742088d | Panagiotis Kanavos | { |
79 | 38ac43a6 | Panagiotis Kanavos | get { return _proceedEvent; } |
80 | 3742088d | Panagiotis Kanavos | } |
81 | 3742088d | Panagiotis Kanavos | |
82 | 3742088d | Panagiotis Kanavos | //Deleted file names are stored in memory so we can check that a file has already been deleted. |
83 | 3742088d | Panagiotis Kanavos | //and avoid sending duplicate delete commands |
84 | 3742088d | Panagiotis Kanavos | readonly ConcurrentDictionary<string, DateTime> _deletedFiles = new ConcurrentDictionary<string, DateTime>(); |
85 | 3742088d | Panagiotis Kanavos | |
86 | 3742088d | Panagiotis Kanavos | public DeleteAgent() |
87 | 3742088d | Panagiotis Kanavos | { |
88 | 3742088d | Panagiotis Kanavos | _deleteAgent = new ActionBlock<CloudDeleteAction>(message => ProcessDelete(message), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4 }); |
89 | 3742088d | Panagiotis Kanavos | |
90 | 3742088d | Panagiotis Kanavos | } |
91 | 3742088d | Panagiotis Kanavos | |
92 | 3742088d | Panagiotis Kanavos | /// <summary> |
93 | 3742088d | Panagiotis Kanavos | /// Processes cloud delete actions |
94 | 3742088d | Panagiotis Kanavos | /// </summary> |
95 | 3742088d | Panagiotis Kanavos | /// <param name="action">The delete action to execute</param> |
96 | 3742088d | Panagiotis Kanavos | /// <returns></returns> |
97 | 3742088d | Panagiotis Kanavos | /// <remarks> |
98 | 3742088d | Panagiotis Kanavos | /// When a file/folder is deleted locally, we must delete it ASAP from the server and block any download |
99 | 3742088d | Panagiotis Kanavos | /// operations that may be in progress. |
100 | 3742088d | Panagiotis Kanavos | /// <para> |
101 | 3742088d | Panagiotis Kanavos | /// A separate agent is used to process deletes because the main agent may be busy with a long operation. |
102 | 3742088d | Panagiotis Kanavos | /// </para> |
103 | 3742088d | Panagiotis Kanavos | /// </remarks> |
104 | 3742088d | Panagiotis Kanavos | private void ProcessDelete(CloudDeleteAction action) |
105 | 3742088d | Panagiotis Kanavos | { |
106 | 3742088d | Panagiotis Kanavos | if (action == null) |
107 | 3742088d | Panagiotis Kanavos | throw new ArgumentNullException("action"); |
108 | 3742088d | Panagiotis Kanavos | if (action.AccountInfo == null) |
109 | 3742088d | Panagiotis Kanavos | throw new ArgumentException("The action.AccountInfo is empty", "action"); |
110 | 3742088d | Panagiotis Kanavos | Contract.EndContractBlock(); |
111 | 3742088d | Panagiotis Kanavos | |
112 | 3742088d | Panagiotis Kanavos | var accountInfo = action.AccountInfo; |
113 | 3742088d | Panagiotis Kanavos | |
114 | 6bcdd8e2 | Panagiotis Kanavos | using (log4net.ThreadContext.Stacks["Operation"].Push("ProcessDelete")) |
115 | 3742088d | Panagiotis Kanavos | { |
116 | 3742088d | Panagiotis Kanavos | Log.InfoFormat("[ACTION] Start Processing {0}", action); |
117 | 3742088d | Panagiotis Kanavos | |
118 | 3742088d | Panagiotis Kanavos | var cloudFile = action.CloudFile; |
119 | 3742088d | Panagiotis Kanavos | |
120 | 3742088d | Panagiotis Kanavos | try |
121 | 3742088d | Panagiotis Kanavos | { |
122 | 3742088d | Panagiotis Kanavos | //Acquire a lock on the deleted file to prevent uploading/downloading operations from the normal |
123 | 3742088d | Panagiotis Kanavos | //agent |
124 | 3742088d | Panagiotis Kanavos | using (NetworkGate.Acquire(action.LocalFile.FullName, NetworkOperation.Deleting)) |
125 | 3742088d | Panagiotis Kanavos | { |
126 | 3742088d | Panagiotis Kanavos | |
127 | 3742088d | Panagiotis Kanavos | //Add the file URL to the deleted files list |
128 | 3742088d | Panagiotis Kanavos | var key = GetFileKey(action.CloudFile); |
129 | 3742088d | Panagiotis Kanavos | _deletedFiles[key] = DateTime.Now; |
130 | 3742088d | Panagiotis Kanavos | |
131 | 38ac43a6 | Panagiotis Kanavos | _proceedEvent.Reset(); |
132 | 3742088d | Panagiotis Kanavos | // and then delete the file from the server |
133 | 3742088d | Panagiotis Kanavos | DeleteCloudFile(accountInfo, cloudFile); |
134 | 3742088d | Panagiotis Kanavos | |
135 | 3742088d | Panagiotis Kanavos | Log.InfoFormat("[ACTION] End Delete {0}:{1}->{2}", action.Action, action.LocalFile, |
136 | 3742088d | Panagiotis Kanavos | action.CloudFile.Name); |
137 | 3742088d | Panagiotis Kanavos | } |
138 | 3742088d | Panagiotis Kanavos | } |
139 | 3742088d | Panagiotis Kanavos | catch (WebException exc) |
140 | 3742088d | Panagiotis Kanavos | { |
141 | 3742088d | Panagiotis Kanavos | Log.ErrorFormat("[WEB ERROR] {0} : {1} -> {2} due to exception\r\n{3}", action.Action, action.LocalFile, action.CloudFile, exc); |
142 | 3742088d | Panagiotis Kanavos | } |
143 | 3742088d | Panagiotis Kanavos | catch (OperationCanceledException) |
144 | 3742088d | Panagiotis Kanavos | { |
145 | 3742088d | Panagiotis Kanavos | throw; |
146 | 3742088d | Panagiotis Kanavos | } |
147 | 3742088d | Panagiotis Kanavos | catch (DirectoryNotFoundException) |
148 | 3742088d | Panagiotis Kanavos | { |
149 | 3742088d | Panagiotis Kanavos | Log.ErrorFormat("{0} : {1} -> {2} failed because the directory was not found.\n Rescheduling a delete", |
150 | 3742088d | Panagiotis Kanavos | action.Action, action.LocalFile, action.CloudFile); |
151 | 3742088d | Panagiotis Kanavos | //Repost a delete action for the missing file |
152 | 3742088d | Panagiotis Kanavos | _deleteAgent.Post(action); |
153 | 3742088d | Panagiotis Kanavos | } |
154 | 3742088d | Panagiotis Kanavos | catch (FileNotFoundException) |
155 | 3742088d | Panagiotis Kanavos | { |
156 | 3742088d | Panagiotis Kanavos | Log.ErrorFormat("{0} : {1} -> {2} failed because the file was not found.\n Rescheduling a delete", |
157 | 3742088d | Panagiotis Kanavos | action.Action, action.LocalFile, action.CloudFile); |
158 | 3742088d | Panagiotis Kanavos | //Post a delete action for the missing file |
159 | 3742088d | Panagiotis Kanavos | _deleteAgent.Post(action); |
160 | 3742088d | Panagiotis Kanavos | } |
161 | 3742088d | Panagiotis Kanavos | catch (Exception exc) |
162 | 3742088d | Panagiotis Kanavos | { |
163 | 3742088d | Panagiotis Kanavos | Log.ErrorFormat("[REQUEUE] {0} : {1} -> {2} due to exception\r\n{3}", |
164 | 3742088d | Panagiotis Kanavos | action.Action, action.LocalFile, action.CloudFile, exc); |
165 | 3742088d | Panagiotis Kanavos | |
166 | 3742088d | Panagiotis Kanavos | _deleteAgent.Post(action); |
167 | 3742088d | Panagiotis Kanavos | } |
168 | 3742088d | Panagiotis Kanavos | finally |
169 | 3742088d | Panagiotis Kanavos | { |
170 | 3742088d | Panagiotis Kanavos | //Set the event when all delete actions are processed |
171 | 3742088d | Panagiotis Kanavos | if (_deleteAgent.InputCount == 0) |
172 | 38ac43a6 | Panagiotis Kanavos | _proceedEvent.Set(); |
173 | 3742088d | Panagiotis Kanavos | |
174 | 3742088d | Panagiotis Kanavos | } |
175 | 3742088d | Panagiotis Kanavos | } |
176 | 3742088d | Panagiotis Kanavos | } |
177 | 3742088d | Panagiotis Kanavos | |
178 | 3742088d | Panagiotis Kanavos | //Returns true if an action concerns a file that was deleted |
179 | 3742088d | Panagiotis Kanavos | public bool IsDeletedFile(CloudAction action) |
180 | 3742088d | Panagiotis Kanavos | { |
181 | 3742088d | Panagiotis Kanavos | //Doesn't work for actions targeting shared files |
182 | 3742088d | Panagiotis Kanavos | if (action.IsShared) |
183 | 3742088d | Panagiotis Kanavos | return false; |
184 | 3742088d | Panagiotis Kanavos | var key = GetFileKey(action.CloudFile); |
185 | 3742088d | Panagiotis Kanavos | DateTime entryDate; |
186 | 3742088d | Panagiotis Kanavos | if (_deletedFiles.TryGetValue(key, out entryDate)) |
187 | 3742088d | Panagiotis Kanavos | { |
188 | 3742088d | Panagiotis Kanavos | //If the delete entry was created after this action, abort the action |
189 | 3742088d | Panagiotis Kanavos | if (entryDate > action.Created) |
190 | 3742088d | Panagiotis Kanavos | return true; |
191 | 3742088d | Panagiotis Kanavos | //Otherwise, remove the stale entry |
192 | 3742088d | Panagiotis Kanavos | _deletedFiles.TryRemove(key, out entryDate); |
193 | 3742088d | Panagiotis Kanavos | } |
194 | 3742088d | Panagiotis Kanavos | return false; |
195 | 3742088d | Panagiotis Kanavos | } |
196 | 3742088d | Panagiotis Kanavos | |
197 | 3742088d | Panagiotis Kanavos | public void Post(CloudDeleteAction action) |
198 | 3742088d | Panagiotis Kanavos | { |
199 | 3742088d | Panagiotis Kanavos | _deleteAgent.Post(action); |
200 | 3742088d | Panagiotis Kanavos | } |
201 | 3742088d | Panagiotis Kanavos | |
202 | 3742088d | Panagiotis Kanavos | private void DeleteCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile) |
203 | 3742088d | Panagiotis Kanavos | { |
204 | 3742088d | Panagiotis Kanavos | if (accountInfo == null) |
205 | 3742088d | Panagiotis Kanavos | throw new ArgumentNullException("accountInfo"); |
206 | 3742088d | Panagiotis Kanavos | if (cloudFile == null) |
207 | 3742088d | Panagiotis Kanavos | throw new ArgumentNullException("cloudFile"); |
208 | 3742088d | Panagiotis Kanavos | |
209 | 3742088d | Panagiotis Kanavos | if (String.IsNullOrWhiteSpace(cloudFile.Container)) |
210 | 3742088d | Panagiotis Kanavos | throw new ArgumentException("Invalid container", "cloudFile"); |
211 | 3742088d | Panagiotis Kanavos | Contract.EndContractBlock(); |
212 | 3742088d | Panagiotis Kanavos | |
213 | 3742088d | Panagiotis Kanavos | var fileAgent = GetFileAgent(accountInfo); |
214 | 3742088d | Panagiotis Kanavos | |
215 | 6bcdd8e2 | Panagiotis Kanavos | using (ThreadContext.Stacks["Operation"].Push("DeleteCloudFile")) |
216 | 3742088d | Panagiotis Kanavos | { |
217 | 3742088d | Panagiotis Kanavos | var fileName = cloudFile.RelativeUrlToFilePath(accountInfo.UserName); |
218 | 3742088d | Panagiotis Kanavos | var info = fileAgent.GetFileSystemInfo(fileName); |
219 | 3742088d | Panagiotis Kanavos | var fullPath = info.FullName.ToLower(); |
220 | 3742088d | Panagiotis Kanavos | |
221 | 3742088d | Panagiotis Kanavos | StatusKeeper.SetFileOverlayStatus(fullPath, FileOverlayStatus.Modified); |
222 | 3742088d | Panagiotis Kanavos | |
223 | 3742088d | Panagiotis Kanavos | var account = cloudFile.Account ?? accountInfo.UserName; |
224 | 3742088d | Panagiotis Kanavos | var container = cloudFile.Container;//?? FolderConstants.PithosContainer; |
225 | 3742088d | Panagiotis Kanavos | |
226 | 3742088d | Panagiotis Kanavos | var client = new CloudFilesClient(accountInfo); |
227 | 3742088d | Panagiotis Kanavos | client.DeleteObject(account, container, cloudFile.Name); |
228 | 3742088d | Panagiotis Kanavos | |
229 | 3742088d | Panagiotis Kanavos | StatusKeeper.ClearFileStatus(fullPath); |
230 | 3742088d | Panagiotis Kanavos | } |
231 | 3742088d | Panagiotis Kanavos | } |
232 | 3742088d | Panagiotis Kanavos | |
233 | 3742088d | Panagiotis Kanavos | |
234 | 3742088d | Panagiotis Kanavos | private static string GetFileKey(ObjectInfo info) |
235 | 3742088d | Panagiotis Kanavos | { |
236 | 3742088d | Panagiotis Kanavos | var key = String.Format("{0}/{1}/{2}", info.Account, info.Container, info.Name); |
237 | 3742088d | Panagiotis Kanavos | return key; |
238 | 3742088d | Panagiotis Kanavos | } |
239 | 3742088d | Panagiotis Kanavos | |
240 | 3742088d | Panagiotis Kanavos | private static FileAgent GetFileAgent(AccountInfo accountInfo) |
241 | 3742088d | Panagiotis Kanavos | { |
242 | 3742088d | Panagiotis Kanavos | return AgentLocator<FileAgent>.Get(accountInfo.AccountPath); |
243 | 3742088d | Panagiotis Kanavos | } |
244 | 3742088d | Panagiotis Kanavos | |
245 | 3742088d | Panagiotis Kanavos | } |
246 | 3742088d | Panagiotis Kanavos | } |