root / trunk / Pithos.Core / Agents / DeleteAgent.cs @ 255f5f86
History | View | Annotate | Download (9.6 kB)
1 | 255f5f86 | Panagiotis Kanavos | #region |
---|---|---|---|
2 | 255f5f86 | Panagiotis Kanavos | /* ----------------------------------------------------------------------- |
3 | 255f5f86 | Panagiotis Kanavos | * <copyright file="DeleteAgent.cs" company="GRNet"> |
4 | 255f5f86 | Panagiotis Kanavos | * |
5 | 255f5f86 | Panagiotis Kanavos | * Copyright 2011-2012 GRNET S.A. All rights reserved. |
6 | 255f5f86 | Panagiotis Kanavos | * |
7 | 255f5f86 | Panagiotis Kanavos | * Redistribution and use in source and binary forms, with or |
8 | 255f5f86 | Panagiotis Kanavos | * without modification, are permitted provided that the following |
9 | 255f5f86 | Panagiotis Kanavos | * conditions are met: |
10 | 255f5f86 | Panagiotis Kanavos | * |
11 | 255f5f86 | Panagiotis Kanavos | * 1. Redistributions of source code must retain the above |
12 | 255f5f86 | Panagiotis Kanavos | * copyright notice, this list of conditions and the following |
13 | 255f5f86 | Panagiotis Kanavos | * disclaimer. |
14 | 255f5f86 | Panagiotis Kanavos | * |
15 | 255f5f86 | Panagiotis Kanavos | * 2. Redistributions in binary form must reproduce the above |
16 | 255f5f86 | Panagiotis Kanavos | * copyright notice, this list of conditions and the following |
17 | 255f5f86 | Panagiotis Kanavos | * disclaimer in the documentation and/or other materials |
18 | 255f5f86 | Panagiotis Kanavos | * provided with the distribution. |
19 | 255f5f86 | Panagiotis Kanavos | * |
20 | 255f5f86 | Panagiotis Kanavos | * |
21 | 255f5f86 | Panagiotis Kanavos | * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
22 | 255f5f86 | Panagiotis Kanavos | * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
23 | 255f5f86 | Panagiotis Kanavos | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
24 | 255f5f86 | Panagiotis Kanavos | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR |
25 | 255f5f86 | Panagiotis Kanavos | * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
26 | 255f5f86 | Panagiotis Kanavos | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
27 | 255f5f86 | Panagiotis Kanavos | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
28 | 255f5f86 | Panagiotis Kanavos | * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
29 | 255f5f86 | Panagiotis Kanavos | * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
30 | 255f5f86 | Panagiotis Kanavos | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
31 | 255f5f86 | Panagiotis Kanavos | * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
32 | 255f5f86 | Panagiotis Kanavos | * POSSIBILITY OF SUCH DAMAGE. |
33 | 255f5f86 | Panagiotis Kanavos | * |
34 | 255f5f86 | Panagiotis Kanavos | * The views and conclusions contained in the software and |
35 | 255f5f86 | Panagiotis Kanavos | * documentation are those of the authors and should not be |
36 | 255f5f86 | Panagiotis Kanavos | * interpreted as representing official policies, either expressed |
37 | 255f5f86 | Panagiotis Kanavos | * or implied, of GRNET S.A. |
38 | 255f5f86 | Panagiotis Kanavos | * </copyright> |
39 | 255f5f86 | Panagiotis Kanavos | * ----------------------------------------------------------------------- |
40 | 255f5f86 | Panagiotis Kanavos | */ |
41 | 255f5f86 | Panagiotis Kanavos | #endregion |
42 | 3742088d | Panagiotis Kanavos | using System.Collections.Concurrent; |
43 | 3742088d | Panagiotis Kanavos | using System.ComponentModel.Composition; |
44 | 3742088d | Panagiotis Kanavos | using System.Diagnostics.Contracts; |
45 | 3742088d | Panagiotis Kanavos | using System.IO; |
46 | 3742088d | Panagiotis Kanavos | using System.Net; |
47 | 3742088d | Panagiotis Kanavos | using System.Threading.Tasks.Dataflow; |
48 | 3742088d | Panagiotis Kanavos | using Pithos.Interfaces; |
49 | 3742088d | Panagiotis Kanavos | using Pithos.Network; |
50 | 3742088d | Panagiotis Kanavos | using System; |
51 | 3742088d | Panagiotis Kanavos | using log4net; |
52 | 3742088d | Panagiotis Kanavos | |
53 | 3742088d | Panagiotis Kanavos | namespace Pithos.Core.Agents |
54 | 3742088d | Panagiotis Kanavos | { |
55 | 3742088d | Panagiotis Kanavos | |
56 | 3742088d | Panagiotis Kanavos | /// <summary> |
57 | 3742088d | Panagiotis Kanavos | /// The Delete Agent is used to delete files from the Pithos server with high priority, |
58 | 3742088d | Panagiotis Kanavos | /// blocking the network agent through the PauseEvent until all pending deletions complete |
59 | 3742088d | Panagiotis Kanavos | /// </summary> |
60 | 3742088d | Panagiotis Kanavos | [Export] |
61 | 3742088d | Panagiotis Kanavos | public class DeleteAgent |
62 | 3742088d | Panagiotis Kanavos | { |
63 | 3742088d | Panagiotis Kanavos | |
64 | 3742088d | Panagiotis Kanavos | [Import] |
65 | 3742088d | Panagiotis Kanavos | public IStatusKeeper StatusKeeper { get; set; } |
66 | 3742088d | Panagiotis Kanavos | |
67 | 3742088d | Panagiotis Kanavos | private static readonly ILog Log = LogManager.GetLogger("DeleteAgent"); |
68 | 3742088d | Panagiotis Kanavos | |
69 | 3742088d | Panagiotis Kanavos | //A separate agent is used to execute delete actions immediatelly; |
70 | 3742088d | Panagiotis Kanavos | private readonly ActionBlock<CloudDeleteAction> _deleteAgent; |
71 | 3742088d | Panagiotis Kanavos | |
72 | 3742088d | Panagiotis Kanavos | //The Pause event stops the network agent to give priority to the deletion agent |
73 | 3742088d | Panagiotis Kanavos | //Initially the event is signalled because we don't need to pause |
74 | 3742088d | Panagiotis Kanavos | private readonly AsyncManualResetEvent _pauseEvent = new AsyncManualResetEvent(true); |
75 | 3742088d | Panagiotis Kanavos | |
76 | 3742088d | Panagiotis Kanavos | public AsyncManualResetEvent PauseEvent |
77 | 3742088d | Panagiotis Kanavos | { |
78 | 3742088d | Panagiotis Kanavos | get { return _pauseEvent; } |
79 | 3742088d | Panagiotis Kanavos | } |
80 | 3742088d | Panagiotis Kanavos | |
81 | 3742088d | Panagiotis Kanavos | //Deleted file names are stored in memory so we can check that a file has already been deleted. |
82 | 3742088d | Panagiotis Kanavos | //and avoid sending duplicate delete commands |
83 | 3742088d | Panagiotis Kanavos | readonly ConcurrentDictionary<string, DateTime> _deletedFiles = new ConcurrentDictionary<string, DateTime>(); |
84 | 3742088d | Panagiotis Kanavos | |
85 | 3742088d | Panagiotis Kanavos | public DeleteAgent() |
86 | 3742088d | Panagiotis Kanavos | { |
87 | 3742088d | Panagiotis Kanavos | _deleteAgent = new ActionBlock<CloudDeleteAction>(message => ProcessDelete(message), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4 }); |
88 | 3742088d | Panagiotis Kanavos | |
89 | 3742088d | Panagiotis Kanavos | } |
90 | 3742088d | Panagiotis Kanavos | |
91 | 3742088d | Panagiotis Kanavos | /// <summary> |
92 | 3742088d | Panagiotis Kanavos | /// Processes cloud delete actions |
93 | 3742088d | Panagiotis Kanavos | /// </summary> |
94 | 3742088d | Panagiotis Kanavos | /// <param name="action">The delete action to execute</param> |
95 | 3742088d | Panagiotis Kanavos | /// <returns></returns> |
96 | 3742088d | Panagiotis Kanavos | /// <remarks> |
97 | 3742088d | Panagiotis Kanavos | /// When a file/folder is deleted locally, we must delete it ASAP from the server and block any download |
98 | 3742088d | Panagiotis Kanavos | /// operations that may be in progress. |
99 | 3742088d | Panagiotis Kanavos | /// <para> |
100 | 3742088d | Panagiotis Kanavos | /// A separate agent is used to process deletes because the main agent may be busy with a long operation. |
101 | 3742088d | Panagiotis Kanavos | /// </para> |
102 | 3742088d | Panagiotis Kanavos | /// </remarks> |
103 | 3742088d | Panagiotis Kanavos | private void ProcessDelete(CloudDeleteAction action) |
104 | 3742088d | Panagiotis Kanavos | { |
105 | 3742088d | Panagiotis Kanavos | if (action == null) |
106 | 3742088d | Panagiotis Kanavos | throw new ArgumentNullException("action"); |
107 | 3742088d | Panagiotis Kanavos | if (action.AccountInfo == null) |
108 | 3742088d | Panagiotis Kanavos | throw new ArgumentException("The action.AccountInfo is empty", "action"); |
109 | 3742088d | Panagiotis Kanavos | Contract.EndContractBlock(); |
110 | 3742088d | Panagiotis Kanavos | |
111 | 3742088d | Panagiotis Kanavos | var accountInfo = action.AccountInfo; |
112 | 3742088d | Panagiotis Kanavos | |
113 | 3742088d | Panagiotis Kanavos | using (log4net.ThreadContext.Stacks["NETWORK"].Push("PROCESS")) |
114 | 3742088d | Panagiotis Kanavos | { |
115 | 3742088d | Panagiotis Kanavos | Log.InfoFormat("[ACTION] Start Processing {0}", action); |
116 | 3742088d | Panagiotis Kanavos | |
117 | 3742088d | Panagiotis Kanavos | var cloudFile = action.CloudFile; |
118 | 3742088d | Panagiotis Kanavos | |
119 | 3742088d | Panagiotis Kanavos | try |
120 | 3742088d | Panagiotis Kanavos | { |
121 | 3742088d | Panagiotis Kanavos | //Acquire a lock on the deleted file to prevent uploading/downloading operations from the normal |
122 | 3742088d | Panagiotis Kanavos | //agent |
123 | 3742088d | Panagiotis Kanavos | using (NetworkGate.Acquire(action.LocalFile.FullName, NetworkOperation.Deleting)) |
124 | 3742088d | Panagiotis Kanavos | { |
125 | 3742088d | Panagiotis Kanavos | |
126 | 3742088d | Panagiotis Kanavos | //Add the file URL to the deleted files list |
127 | 3742088d | Panagiotis Kanavos | var key = GetFileKey(action.CloudFile); |
128 | 3742088d | Panagiotis Kanavos | _deletedFiles[key] = DateTime.Now; |
129 | 3742088d | Panagiotis Kanavos | |
130 | 3742088d | Panagiotis Kanavos | _pauseEvent.Reset(); |
131 | 3742088d | Panagiotis Kanavos | // and then delete the file from the server |
132 | 3742088d | Panagiotis Kanavos | DeleteCloudFile(accountInfo, cloudFile); |
133 | 3742088d | Panagiotis Kanavos | |
134 | 3742088d | Panagiotis Kanavos | Log.InfoFormat("[ACTION] End Delete {0}:{1}->{2}", action.Action, action.LocalFile, |
135 | 3742088d | Panagiotis Kanavos | action.CloudFile.Name); |
136 | 3742088d | Panagiotis Kanavos | } |
137 | 3742088d | Panagiotis Kanavos | } |
138 | 3742088d | Panagiotis Kanavos | catch (WebException exc) |
139 | 3742088d | Panagiotis Kanavos | { |
140 | 3742088d | Panagiotis Kanavos | Log.ErrorFormat("[WEB ERROR] {0} : {1} -> {2} due to exception\r\n{3}", action.Action, action.LocalFile, action.CloudFile, exc); |
141 | 3742088d | Panagiotis Kanavos | } |
142 | 3742088d | Panagiotis Kanavos | catch (OperationCanceledException) |
143 | 3742088d | Panagiotis Kanavos | { |
144 | 3742088d | Panagiotis Kanavos | throw; |
145 | 3742088d | Panagiotis Kanavos | } |
146 | 3742088d | Panagiotis Kanavos | catch (DirectoryNotFoundException) |
147 | 3742088d | Panagiotis Kanavos | { |
148 | 3742088d | Panagiotis Kanavos | Log.ErrorFormat("{0} : {1} -> {2} failed because the directory was not found.\n Rescheduling a delete", |
149 | 3742088d | Panagiotis Kanavos | action.Action, action.LocalFile, action.CloudFile); |
150 | 3742088d | Panagiotis Kanavos | //Repost a delete action for the missing file |
151 | 3742088d | Panagiotis Kanavos | _deleteAgent.Post(action); |
152 | 3742088d | Panagiotis Kanavos | } |
153 | 3742088d | Panagiotis Kanavos | catch (FileNotFoundException) |
154 | 3742088d | Panagiotis Kanavos | { |
155 | 3742088d | Panagiotis Kanavos | Log.ErrorFormat("{0} : {1} -> {2} failed because the file was not found.\n Rescheduling a delete", |
156 | 3742088d | Panagiotis Kanavos | action.Action, action.LocalFile, action.CloudFile); |
157 | 3742088d | Panagiotis Kanavos | //Post a delete action for the missing file |
158 | 3742088d | Panagiotis Kanavos | _deleteAgent.Post(action); |
159 | 3742088d | Panagiotis Kanavos | } |
160 | 3742088d | Panagiotis Kanavos | catch (Exception exc) |
161 | 3742088d | Panagiotis Kanavos | { |
162 | 3742088d | Panagiotis Kanavos | Log.ErrorFormat("[REQUEUE] {0} : {1} -> {2} due to exception\r\n{3}", |
163 | 3742088d | Panagiotis Kanavos | action.Action, action.LocalFile, action.CloudFile, exc); |
164 | 3742088d | Panagiotis Kanavos | |
165 | 3742088d | Panagiotis Kanavos | _deleteAgent.Post(action); |
166 | 3742088d | Panagiotis Kanavos | } |
167 | 3742088d | Panagiotis Kanavos | finally |
168 | 3742088d | Panagiotis Kanavos | { |
169 | 3742088d | Panagiotis Kanavos | //Set the event when all delete actions are processed |
170 | 3742088d | Panagiotis Kanavos | if (_deleteAgent.InputCount == 0) |
171 | 3742088d | Panagiotis Kanavos | _pauseEvent.Set(); |
172 | 3742088d | Panagiotis Kanavos | |
173 | 3742088d | Panagiotis Kanavos | } |
174 | 3742088d | Panagiotis Kanavos | } |
175 | 3742088d | Panagiotis Kanavos | } |
176 | 3742088d | Panagiotis Kanavos | |
177 | 3742088d | Panagiotis Kanavos | //Returns true if an action concerns a file that was deleted |
178 | 3742088d | Panagiotis Kanavos | public bool IsDeletedFile(CloudAction action) |
179 | 3742088d | Panagiotis Kanavos | { |
180 | 3742088d | Panagiotis Kanavos | //Doesn't work for actions targeting shared files |
181 | 3742088d | Panagiotis Kanavos | if (action.IsShared) |
182 | 3742088d | Panagiotis Kanavos | return false; |
183 | 3742088d | Panagiotis Kanavos | var key = GetFileKey(action.CloudFile); |
184 | 3742088d | Panagiotis Kanavos | DateTime entryDate; |
185 | 3742088d | Panagiotis Kanavos | if (_deletedFiles.TryGetValue(key, out entryDate)) |
186 | 3742088d | Panagiotis Kanavos | { |
187 | 3742088d | Panagiotis Kanavos | //If the delete entry was created after this action, abort the action |
188 | 3742088d | Panagiotis Kanavos | if (entryDate > action.Created) |
189 | 3742088d | Panagiotis Kanavos | return true; |
190 | 3742088d | Panagiotis Kanavos | //Otherwise, remove the stale entry |
191 | 3742088d | Panagiotis Kanavos | _deletedFiles.TryRemove(key, out entryDate); |
192 | 3742088d | Panagiotis Kanavos | } |
193 | 3742088d | Panagiotis Kanavos | return false; |
194 | 3742088d | Panagiotis Kanavos | } |
195 | 3742088d | Panagiotis Kanavos | |
196 | 3742088d | Panagiotis Kanavos | public void Post(CloudDeleteAction action) |
197 | 3742088d | Panagiotis Kanavos | { |
198 | 3742088d | Panagiotis Kanavos | _deleteAgent.Post(action); |
199 | 3742088d | Panagiotis Kanavos | } |
200 | 3742088d | Panagiotis Kanavos | |
201 | 3742088d | Panagiotis Kanavos | private void DeleteCloudFile(AccountInfo accountInfo, ObjectInfo cloudFile) |
202 | 3742088d | Panagiotis Kanavos | { |
203 | 3742088d | Panagiotis Kanavos | if (accountInfo == null) |
204 | 3742088d | Panagiotis Kanavos | throw new ArgumentNullException("accountInfo"); |
205 | 3742088d | Panagiotis Kanavos | if (cloudFile == null) |
206 | 3742088d | Panagiotis Kanavos | throw new ArgumentNullException("cloudFile"); |
207 | 3742088d | Panagiotis Kanavos | |
208 | 3742088d | Panagiotis Kanavos | if (String.IsNullOrWhiteSpace(cloudFile.Container)) |
209 | 3742088d | Panagiotis Kanavos | throw new ArgumentException("Invalid container", "cloudFile"); |
210 | 3742088d | Panagiotis Kanavos | Contract.EndContractBlock(); |
211 | 3742088d | Panagiotis Kanavos | |
212 | 3742088d | Panagiotis Kanavos | var fileAgent = GetFileAgent(accountInfo); |
213 | 3742088d | Panagiotis Kanavos | |
214 | 3742088d | Panagiotis Kanavos | using (ThreadContext.Stacks["DeleteCloudFile"].Push("Delete")) |
215 | 3742088d | Panagiotis Kanavos | { |
216 | 3742088d | Panagiotis Kanavos | var fileName = cloudFile.RelativeUrlToFilePath(accountInfo.UserName); |
217 | 3742088d | Panagiotis Kanavos | var info = fileAgent.GetFileSystemInfo(fileName); |
218 | 3742088d | Panagiotis Kanavos | var fullPath = info.FullName.ToLower(); |
219 | 3742088d | Panagiotis Kanavos | |
220 | 3742088d | Panagiotis Kanavos | StatusKeeper.SetFileOverlayStatus(fullPath, FileOverlayStatus.Modified); |
221 | 3742088d | Panagiotis Kanavos | |
222 | 3742088d | Panagiotis Kanavos | var account = cloudFile.Account ?? accountInfo.UserName; |
223 | 3742088d | Panagiotis Kanavos | var container = cloudFile.Container;//?? FolderConstants.PithosContainer; |
224 | 3742088d | Panagiotis Kanavos | |
225 | 3742088d | Panagiotis Kanavos | var client = new CloudFilesClient(accountInfo); |
226 | 3742088d | Panagiotis Kanavos | client.DeleteObject(account, container, cloudFile.Name); |
227 | 3742088d | Panagiotis Kanavos | |
228 | 3742088d | Panagiotis Kanavos | StatusKeeper.ClearFileStatus(fullPath); |
229 | 3742088d | Panagiotis Kanavos | } |
230 | 3742088d | Panagiotis Kanavos | } |
231 | 3742088d | Panagiotis Kanavos | |
232 | 3742088d | Panagiotis Kanavos | |
233 | 3742088d | Panagiotis Kanavos | private static string GetFileKey(ObjectInfo info) |
234 | 3742088d | Panagiotis Kanavos | { |
235 | 3742088d | Panagiotis Kanavos | var key = String.Format("{0}/{1}/{2}", info.Account, info.Container, info.Name); |
236 | 3742088d | Panagiotis Kanavos | return key; |
237 | 3742088d | Panagiotis Kanavos | } |
238 | 3742088d | Panagiotis Kanavos | |
239 | 3742088d | Panagiotis Kanavos | private static FileAgent GetFileAgent(AccountInfo accountInfo) |
240 | 3742088d | Panagiotis Kanavos | { |
241 | 3742088d | Panagiotis Kanavos | return AgentLocator<FileAgent>.Get(accountInfo.AccountPath); |
242 | 3742088d | Panagiotis Kanavos | } |
243 | 3742088d | Panagiotis Kanavos | |
244 | 3742088d | Panagiotis Kanavos | } |
245 | 3742088d | Panagiotis Kanavos | } |