Statistics
| Branch: | Revision:

root / trunk / Pithos.Core / Agents / Uploader.cs @ 8f44fd3a

History | View | Annotate | Download (13.1 kB)

1
´╗┐using System;
2
using System.ComponentModel.Composition;
3
using System.Diagnostics;
4
using System.Diagnostics.Contracts;
5
using System.IO;
6
using System.Net;
7
using System.Reflection;
8
using System.Threading.Tasks;
9
using Pithos.Interfaces;
10
using Pithos.Network;
11
using log4net;
12

    
13
namespace Pithos.Core.Agents
14
{
15
    [Export(typeof(Uploader))]
16
    public class Uploader
17
    {
18
        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
19

    
20
        [Import]
21
        private IStatusKeeper StatusKeeper { get; set; }
22

    
23
        
24
        public IStatusNotification StatusNotification { get; set; }
25

    
26
        public async Task UploadCloudFile(CloudAction action)
27
        {
28
            if (action == null)
29
                throw new ArgumentNullException("action");
30
            Contract.EndContractBlock();
31

    
32
            using (ThreadContext.Stacks["Operation"].Push("UploadCloudFile"))
33
            {
34
                try
35
                {
36
                    var fileInfo = action.LocalFile;
37

    
38
                    if (fileInfo.Extension.Equals("ignore", StringComparison.InvariantCultureIgnoreCase))
39
                        return;
40

    
41
                    //Try to load the action's local state, if it is empty
42
                    if (action.FileState == null)
43
                        action.FileState = StatusKeeper.GetStateByFilePath(fileInfo.FullName);
44
                    if (action.FileState == null)
45
                    {
46
                        Log.WarnFormat("File [{0}] has no local state. It was probably created by a download action", fileInfo.FullName);
47
                        return;
48
                    }
49

    
50
                    var latestState = action.FileState;
51

    
52
                    //Do not upload files in conflict
53
                    if (latestState.FileStatus == FileStatus.Conflict)
54
                    {
55
                        Log.InfoFormat("Skipping file in conflict [{0}]", fileInfo.FullName);
56
                        return;
57
                    }
58
                    //Do not upload files when we have no permission
59
                    if (latestState.FileStatus == FileStatus.Forbidden)
60
                    {
61
                        Log.InfoFormat("Skipping forbidden file [{0}]", fileInfo.FullName);
62
                        return;
63
                    }
64

    
65
                    //Are we targeting our own account or a sharer account?
66
                    var relativePath = fileInfo.AsRelativeTo(action.AccountInfo.AccountPath);
67
                    var accountInfo = relativePath.StartsWith(FolderConstants.OthersFolder) 
68
                                                  ? GetSharerAccount(relativePath, action.AccountInfo) 
69
                                                  : action.AccountInfo;
70

    
71

    
72

    
73
                    var fullFileName = fileInfo.GetProperCapitalization();
74
                    using (var gate = NetworkGate.Acquire(fullFileName, NetworkOperation.Uploading))
75
                    {
76
                        //Abort if the file is already being uploaded or downloaded
77
                        if (gate.Failed)
78
                            return;
79

    
80
                        var cloudFile = action.CloudFile;
81
                        var account = cloudFile.Account ?? accountInfo.UserName;
82
                        try
83
                        {
84

    
85
                            var client = new CloudFilesClient(accountInfo);
86
                            //Even if GetObjectInfo times out, we can proceed with the upload            
87
                            var cloudInfo = client.GetObjectInfo(account, cloudFile.Container, cloudFile.Name);
88

    
89
                            //If this is a read-only file, do not upload changes
90
                            if (!cloudInfo.IsWritable(action.AccountInfo.UserName))
91
                                return;
92

    
93
                            if (fileInfo is DirectoryInfo)
94
                            {
95
                                //If the directory doesn't exist the Hash property will be empty
96
                                if (String.IsNullOrWhiteSpace(cloudInfo.Hash))
97
                                    //Go on and create the directory
98
                                    await client.PutObject(account, cloudFile.Container, cloudFile.Name, fullFileName,
99
                                                         String.Empty, "application/directory");
100
                            }
101
                            else
102
                            {
103

    
104
                                var cloudHash = cloudInfo.Hash.ToLower();
105

    
106
                                string topHash;
107
                                TreeHash treeHash;
108
                                using(StatusNotification.GetNotifier("Hashing {0} for Upload", "Finished hashing {0}",fileInfo.Name))
109
                                {
110
                                    treeHash = action.TreeHash.Value;
111
                                    topHash = treeHash.TopHash.ToHashString();
112
                                }
113

    
114
                                //If the file hashes match, abort the upload
115
                                if (cloudInfo != ObjectInfo.Empty && topHash == cloudHash)
116
                                {
117
                                    //but store any metadata changes 
118
                                    StatusKeeper.StoreInfo(fullFileName, cloudInfo);
119
                                    Log.InfoFormat("Skip upload of {0}, hashes match", fullFileName);
120
                                    return;
121
                                }
122

    
123

    
124
                                //Mark the file as modified while we upload it
125
                                StatusKeeper.SetFileOverlayStatus(fullFileName, FileOverlayStatus.Modified);
126
                                //And then upload it
127

    
128
                                //Upload even small files using the Hashmap. The server may already contain
129
                                //the relevant block                                
130

    
131
                                await UploadWithHashMap(accountInfo, cloudFile, fileInfo as FileInfo, cloudFile.Name, treeHash);
132
                            }
133
                            //If everything succeeds, change the file and overlay status to normal
134
                            StatusKeeper.SetFileState(fullFileName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");
135
                        }
136
                        catch (WebException exc)
137
                        {
138
                            var response = (exc.Response as HttpWebResponse);
139
                            if (response == null)
140
                                throw;
141
                            if (response.StatusCode == HttpStatusCode.Forbidden)
142
                            {
143
                                StatusKeeper.SetFileState(fileInfo.FullName, FileStatus.Forbidden, FileOverlayStatus.Conflict, "Forbidden");
144
                                
145
                            }
146
                            else
147
                                //In any other case, propagate the error
148
                                throw;
149
                        }
150
                    }
151
                    //Notify the Shell to update the overlays
152
                    NativeMethods.RaiseChangeNotification(fullFileName);
153
                    StatusNotification.NotifyChangedFile(fullFileName);
154
                }
155
                catch (AggregateException ex)
156
                {
157
                    var exc = ex.InnerException as WebException;
158
                    if (exc == null)
159
                        throw ex.InnerException;
160
                    if (HandleUploadWebException(action, exc))
161
                        return;
162
                    throw;
163
                }
164
                catch (WebException ex)
165
                {
166
                    if (HandleUploadWebException(action, ex))
167
                        return;
168
                    throw;
169
                }
170
                catch (Exception ex)
171
                {
172
                    Log.Error("Unexpected error while uploading file", ex);
173
                    throw;
174
                }
175
            }
176
        }
177

    
178
        private static AccountInfo GetSharerAccount(string relativePath, AccountInfo accountInfo)
179
        {
180
            var parts = relativePath.Split('\\');
181
            var accountName = parts[1];
182
            var oldName = accountInfo.UserName;
183
            var absoluteUri = accountInfo.StorageUri.AbsoluteUri;
184
            var nameIndex = absoluteUri.IndexOf(oldName, StringComparison.Ordinal);
185
            var root = absoluteUri.Substring(0, nameIndex);
186

    
187
            accountInfo = new AccountInfo
188
                              {
189
                                  UserName = accountName,
190
                                  AccountPath = Path.Combine(accountInfo.AccountPath, parts[0], parts[1]),
191
                                  StorageUri = new Uri(root + accountName),
192
                                  BlockHash = accountInfo.BlockHash,
193
                                  BlockSize = accountInfo.BlockSize,
194
                                  Token = accountInfo.Token
195
                              };
196
            return accountInfo;
197
        }
198

    
199

    
200
        public async Task UploadWithHashMap(AccountInfo accountInfo, ObjectInfo cloudFile, FileInfo fileInfo, string url, TreeHash treeHash)
201
        {
202
            if (accountInfo == null)
203
                throw new ArgumentNullException("accountInfo");
204
            if (cloudFile == null)
205
                throw new ArgumentNullException("cloudFile");
206
            if (fileInfo == null)
207
                throw new ArgumentNullException("fileInfo");
208
            if (String.IsNullOrWhiteSpace(url))
209
                throw new ArgumentNullException(url);
210
            if (treeHash == null)
211
                throw new ArgumentNullException("treeHash");
212
            if (String.IsNullOrWhiteSpace(cloudFile.Container))
213
                throw new ArgumentException("Invalid container", "cloudFile");
214
            Contract.EndContractBlock();
215

    
216
            using (StatusNotification.GetNotifier("Uploading {0}", "Finished Uploading {0}", fileInfo.Name))
217
            {
218

    
219
                var fullFileName = fileInfo.GetProperCapitalization();
220

    
221
                var account = cloudFile.Account ?? accountInfo.UserName;
222
                var container = cloudFile.Container;
223

    
224
                var client = new CloudFilesClient(accountInfo);
225
                //Send the hashmap to the server            
226
                var missingHashes = await client.PutHashMap(account, container, url, treeHash);
227
                int block = 0;
228
                ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);
229
                //If the server returns no missing hashes, we are done
230
                while (missingHashes.Count > 0)
231
                {
232

    
233
                    var buffer = new byte[accountInfo.BlockSize];
234
                    foreach (var missingHash in missingHashes)
235
                    {
236
                        //Find the proper block
237
                        var blockIndex = treeHash.HashDictionary[missingHash];
238
                        long offset = blockIndex*accountInfo.BlockSize;
239

    
240
                        var read = fileInfo.Read(buffer, offset, accountInfo.BlockSize);
241

    
242
                        try
243
                        {
244
                            //And upload the block                
245
                            await client.PostBlock(account, container, buffer, 0, read);
246
                            Log.InfoFormat("[BLOCK] Block {0} of {1} uploaded", blockIndex, fullFileName);
247
                        }
248
                        catch (Exception exc)
249
                        {
250
                            Log.Error(String.Format("Uploading block {0} of {1}", blockIndex, fullFileName), exc);
251
                        }
252
                        ReportUploadProgress(fileInfo.Name, block++, missingHashes.Count, fileInfo.Length);
253
                    }
254

    
255
                    //Repeat until there are no more missing hashes                
256
                    missingHashes = await client.PutHashMap(account, container, url, treeHash);
257
                }
258

    
259
                ReportUploadProgress(fileInfo.Name, missingHashes.Count, missingHashes.Count, fileInfo.Length);
260
            }
261
        }
262

    
263
        private void ReportUploadProgress(string fileName, int block, int totalBlocks, long fileSize)
264
        {
265
            StatusNotification.Notify(totalBlocks == 0
266
                                          ? new ProgressNotification(fileName, "Uploading", 1, 1, fileSize)
267
                                          : new ProgressNotification(fileName, "Uploading", block, totalBlocks, fileSize));
268
        }
269

    
270

    
271
        private bool HandleUploadWebException(CloudAction action, WebException exc)
272
        {
273
            var response = exc.Response as HttpWebResponse;
274
            if (response == null)
275
                throw exc;
276
            if (response.StatusCode == HttpStatusCode.Unauthorized)
277
            {
278
                Log.Error("Not allowed to upload file", exc);
279
                var message = String.Format("Not allowed to uplad file {0}", action.LocalFile.FullName);
280
                StatusKeeper.SetFileState(action.LocalFile.FullName, FileStatus.Unchanged, FileOverlayStatus.Normal, "");
281
                StatusNotification.NotifyChange(message, TraceLevel.Warning);
282
                return true;
283
            }
284
            return false;
285
        }
286
    }
287
}