Statistics
| Branch: | Revision:

root / trunk / Pithos.Network / CloudFilesClient.cs @ 0eea575a

History | View | Annotate | Download (23.9 kB)

1
using System;
2
using System.Collections.Generic;
3
using System.ComponentModel.Composition;
4
using System.Diagnostics;
5
using System.Diagnostics.Contracts;
6
using System.IO;
7
using System.Linq;
8
using System.Net;
9
using System.Security.Cryptography;
10
using System.Text;
11
using System.Threading.Tasks;
12
using Hammock;
13
using Hammock.Caching;
14
using Hammock.Retries;
15
using Hammock.Serialization;
16
using Hammock.Tasks;
17
using Hammock.Web;
18
using Newtonsoft.Json;
19
using Pithos.Interfaces;
20

    
21
namespace Pithos.Network
22
{
23
    [Export(typeof(ICloudClient))]
24
    public class CloudFilesClient:ICloudClient
25
    {
26
        string _rackSpaceAuthUrl = "https://auth.api.rackspacecloud.com";
27
        private string _pithosAuthUrl = "https://pithos.grnet.gr";
28

    
29
        private RestClient _client;
30
        private readonly TimeSpan _shortTimeout = TimeSpan.FromSeconds(10);
31
        private readonly int _retries = 5;
32
        private RetryPolicy _retryPolicy;
33
        public string ApiKey { get; set; }
34
        public string UserName { get; set; }
35
        public Uri StorageUrl { get; set; }
36
        public string Token { get; set; }
37
        public Uri Proxy { get; set; }
38

    
39
        public double DownloadPercentLimit { get; set; }
40
        public double UploadPercentLimit { get; set; }
41
        
42
        public string AuthUrl
43
        {
44
            get { return UsePithos ? _pithosAuthUrl : _rackSpaceAuthUrl; }
45
        }
46
 
47
        public string VersionPath
48
        {
49
            get { return UsePithos ? "v1" : "v1.0"; }
50
        }
51

    
52
        public bool UsePithos { get; set; }
53

    
54
        public void Authenticate(string userName,string apiKey)
55
        {
56
            Trace.TraceInformation("[AUTHENTICATE] Start for {0}", userName);
57
            if (String.IsNullOrWhiteSpace(userName))
58
                throw new ArgumentNullException("userName", "The userName property can't be empty");
59
            if (String.IsNullOrWhiteSpace(apiKey))
60
                throw new ArgumentNullException("apiKey", "The apiKey property can't be empty");
61

    
62
            UserName = userName;
63
            ApiKey = apiKey;
64
            
65
            var proxy = Proxy != null ? Proxy.ToString() : null;
66
            if (UsePithos)
67
            {
68
                Token = "0000";
69
                string storageUrl = String.Format("{0}/{1}/{2}", AuthUrl, VersionPath, UserName);
70
                StorageUrl = new Uri(storageUrl);
71
            }
72
            else
73
            {
74

    
75
                string authUrl = String.Format("{0}/{1}", AuthUrl, VersionPath);
76
                var authClient = new RestClient {Path = authUrl, Proxy = proxy};                
77

    
78
                authClient.AddHeader("X-Auth-User", UserName);
79
                authClient.AddHeader("X-Auth-Key", ApiKey);
80

    
81
                var response = authClient.Request();
82

    
83
                ThrowIfNotStatusOK(response, "Authentication failed");
84

    
85
                var keys = response.Headers.AllKeys.AsQueryable();
86

    
87
                string storageUrl = GetHeaderValue("X-Storage-Url", response, keys);
88
                if (String.IsNullOrWhiteSpace(storageUrl))
89
                    throw new InvalidOperationException("Failed to obtain storage url");
90
                StorageUrl = new Uri(storageUrl);
91

    
92
                var token = GetHeaderValue("X-Auth-Token", response, keys);
93
                if (String.IsNullOrWhiteSpace(token))
94
                    throw new InvalidOperationException("Failed to obtain token url");
95
                Token = token;
96
            }
97

    
98
            _retryPolicy = new RetryPolicy { RetryCount = _retries };
99
            _retryPolicy.RetryConditions.Add(new TimeoutRetryCondition());
100

    
101
            _client = new RestClient { Authority = StorageUrl.AbsoluteUri, Path = UserName, Proxy = proxy };
102
            _client.FileProgress += OnFileProgress;
103
            
104
            _client.AddHeader("X-Auth-Token", Token);
105
            /*if (UsePithos)
106
            {
107
                _client.AddHeader("X-Auth-User", UserName);
108
                _client.AddHeader("X-Auth-Key",ApiKey);                
109
            }*/
110

    
111
            Trace.TraceInformation("[AUTHENTICATE] End for {0}", userName);
112
        }
113

    
114
        private void OnFileProgress(object sender, FileProgressEventArgs e)
115
        {
116
            Trace.TraceInformation("[PROGRESS] {0} {1:p} {2} of {3}",e.FileName,(double)e.BytesWritten/e.TotalBytes, e.BytesWritten,e.TotalBytes);            
117
        }
118

    
119
        public IList<ContainerInfo> ListContainers()
120
        {                        
121
            //Workaround for Hammock quirk: Hammock always
122
            //appends a / unless a Path is specified.
123
            
124
            //Create a request with a complete path
125
            var request = new RestRequest { Path = StorageUrl.ToString(), RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
126
            request.AddParameter("format","json");
127
            //Create a client clone
128
            var client = new RestClient{Proxy=Proxy.ToString()};
129
            foreach (var header in _client.GetAllHeaders())
130
            {
131
                client.AddHeader(header.Name,header.Value);
132
            }            
133

    
134
            var response = client.Request(request);
135

    
136
            if (response.StatusCode == HttpStatusCode.NoContent)
137
                return new List<ContainerInfo>();
138

    
139
            ThrowIfNotStatusOK(response, "List Containers failed");
140

    
141

    
142
            var infos=JsonConvert.DeserializeObject<IList<ContainerInfo>>(response.Content);
143
            
144
            return infos;
145
        }
146

    
147
        public IList<ObjectInfo> ListObjects(string container)
148
        {
149
            if (String.IsNullOrWhiteSpace(container))
150
                throw new ArgumentNullException("container", "The container property can't be empty");
151

    
152
            Trace.TraceInformation("[START] ListObjects");
153

    
154
            var request = new RestRequest { Path = container, RetryPolicy = _retryPolicy, Timeout = TimeSpan.FromMinutes(1) };
155
            request.AddParameter("format", "json");
156
            var response = _client.Request(request);
157
            
158
            var infos = InfosFromContent(response);
159

    
160
            Trace.TraceInformation("[END] ListObjects");
161
            return infos;
162
        }
163

    
164

    
165

    
166
        public IList<ObjectInfo> ListObjects(string container,string folder)
167
        {
168
            if (String.IsNullOrWhiteSpace(container))
169
                throw new ArgumentNullException("container", "The container property can't be empty");
170

    
171
            Trace.TraceInformation("[START] ListObjects");
172

    
173
            var request = new RestRequest { Path = container,RetryPolicy = _retryPolicy, Timeout = TimeSpan.FromMinutes(1) };
174
            request.AddParameter("format", "json");
175
            request.AddParameter("path", folder);
176
            var response = _client.Request(request);
177
            
178
            var infos = InfosFromContent(response);
179

    
180
            Trace.TraceInformation("[END] ListObjects");
181
            return infos;
182
        }
183

    
184
        private static IList<ObjectInfo> InfosFromContent(RestResponse response)
185
        {
186
            if (response.TimedOut)
187
                return new List<ObjectInfo>();
188

    
189
            if (response.StatusCode == 0)
190
                return new List<ObjectInfo>();
191

    
192
            if (response.StatusCode == HttpStatusCode.NoContent)
193
                return new List<ObjectInfo>();
194

    
195

    
196
            var statusCode = (int)response.StatusCode;
197
            if (statusCode < 200 || statusCode >= 300)
198
            {
199
                Trace.TraceWarning("ListObjects failed with code {0} - {1}", response.StatusCode, response.StatusDescription);
200
                return new List<ObjectInfo>();
201
            }
202

    
203
            var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(response.Content);
204
            return infos;
205
        }
206

    
207
        public bool ContainerExists(string container)
208
        {
209
            if (String.IsNullOrWhiteSpace(container))
210
                throw new ArgumentNullException("container", "The container property can't be empty");
211

    
212
            var request = new RestRequest { Path = container, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
213
            var response = _client.Request(request);
214

    
215
            switch(response.StatusCode)
216
            {
217
                case HttpStatusCode.NoContent:
218
                    return true;
219
                case HttpStatusCode.NotFound:
220
                    return false;                    
221
                default:
222
                    throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode));
223
            }
224
        }
225

    
226
        public bool ObjectExists(string container,string objectName)
227
        {
228
            if (String.IsNullOrWhiteSpace(container))
229
                throw new ArgumentNullException("container", "The container property can't be empty");
230
            if (String.IsNullOrWhiteSpace(objectName))
231
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
232

    
233

    
234
            var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head,RetryPolicy = _retryPolicy, Timeout = _shortTimeout };
235
            var response = _client.Request(request);
236

    
237
            switch (response.StatusCode)
238
            {
239
                case HttpStatusCode.OK:
240
                case HttpStatusCode.NoContent:
241
                    return true;
242
                case HttpStatusCode.NotFound:
243
                    return false;
244
                default:
245
                    throw new WebException(String.Format("ObjectExists failed with unexpected status code {0}", response.StatusCode));
246
            }
247
            
248
        }
249

    
250
        public ObjectInfo GetObjectInfo(string container, string objectName)
251
        {
252
            if (String.IsNullOrWhiteSpace(container))
253
                throw new ArgumentNullException("container", "The container property can't be empty");
254
            if (String.IsNullOrWhiteSpace(objectName))
255
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
256

    
257

    
258
            var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
259
            var response = _client.Request(request);
260

    
261
            if (response.TimedOut)
262
                return ObjectInfo.Empty;
263

    
264
            switch (response.StatusCode)
265
            {
266
                case HttpStatusCode.OK:
267
                case HttpStatusCode.NoContent:
268
                    var keys = response.Headers.AllKeys.AsQueryable();
269
                    return new ObjectInfo
270
                               {
271
                                   Name = objectName,
272
                                   Bytes = long.Parse(GetHeaderValue("Content-Length", response, keys)),
273
                                   Hash = GetHeaderValue("ETag", response, keys),
274
                                   Content_Type = GetHeaderValue("Content-Type", response, keys)
275
                               };
276
                case HttpStatusCode.NotFound:
277
                    return ObjectInfo.Empty;
278
                default:
279
                    if (request.RetryState.RepeatCount > 0)
280
                    {
281
                        Trace.TraceWarning("[RETRY FAIL] GetObjectInfo for {0} failed after {1} retries",
282
                                                      objectName, request.RetryState.RepeatCount);
283
                        return ObjectInfo.Empty;
284
                    }
285
                    if (response.InnerException != null)
286
                        throw new WebException(String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", objectName, response.StatusCode), response.InnerException);
287
                    throw new WebException(String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}", objectName, response.StatusCode));
288
            }
289
        }
290

    
291
        public void CreateFolder(string container, string folder)
292
        {
293
            if (String.IsNullOrWhiteSpace(container))
294
                throw new ArgumentNullException("container", "The container property can't be empty");
295
            if (String.IsNullOrWhiteSpace(folder))
296
                throw new ArgumentNullException("folder", "The folder property can't be empty");
297

    
298
            var folderUrl=String.Format("{0}/{1}",container,folder);
299
            var request = new RestRequest { Path = folderUrl, Method = WebMethod.Put, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
300
            request.AddHeader("Content-Type", @"application/directory");
301
            request.AddHeader("Content-Length", "0");
302

    
303
            var response = _client.Request(request);
304

    
305
            if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted)
306
                throw new WebException(String.Format("CreateFolder failed with unexpected status code {0}", response.StatusCode));
307

    
308
        }
309

    
310
        public ContainerInfo GetContainerInfo(string container)
311
        {
312
            if (String.IsNullOrWhiteSpace(container))
313
                throw new ArgumentNullException("container", "The container property can't be empty");
314

    
315
            var request = new RestRequest { Path = container, Method = WebMethod.Head, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
316
            var response = _client.Request(request);
317

    
318
            switch(response.StatusCode)
319
            {
320
                case HttpStatusCode.NoContent:
321
                    var keys = response.Headers.AllKeys.AsQueryable();
322
                    var containerInfo = new ContainerInfo
323
                                            {
324
                                                Name = container,
325
                                                Count =long.Parse(GetHeaderValue("X-Container-Object-Count", response, keys)),
326
                                                Bytes =long.Parse(GetHeaderValue("X-Container-Bytes-Used", response, keys))
327
                                            };
328
                    return containerInfo;
329
                case HttpStatusCode.NotFound:
330
                    return ContainerInfo.Empty;                    
331
                default:
332
                    throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}",response.StatusCode));
333
            }
334
        }
335

    
336
        public void CreateContainer(string container)
337
        {
338
            if (String.IsNullOrWhiteSpace(container))
339
                throw new ArgumentNullException("container", "The container property can't be empty");
340

    
341
            var request = new RestRequest { Path = container, Method = WebMethod.Put, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
342
            
343
            var response = _client.Request(request);
344
                        
345
            if (response.StatusCode!=HttpStatusCode.Created && response.StatusCode!=HttpStatusCode.Accepted )
346
                    throw new WebException(String.Format("ContainerExists failed with unexpected status code {0}", response.StatusCode));
347
        }
348

    
349
        public void DeleteContainer(string container)
350
        {
351
            if (String.IsNullOrWhiteSpace(container))
352
                throw new ArgumentNullException("container", "The container property can't be empty");
353

    
354
            var request = new RestRequest { Path = container, Method = WebMethod.Delete, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
355
            var response = _client.Request(request);
356

    
357
            if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent)
358
                return;
359
            else
360
                throw new WebException(String.Format("DeleteContainer failed with unexpected status code {0}", response.StatusCode));
361

    
362
        }
363

    
364
        /// <summary>
365
        /// 
366
        /// </summary>
367
        /// <param name="container"></param>
368
        /// <param name="objectName"></param>
369
        /// <returns></returns>
370
        /// <remarks>>This method should have no timeout or a very long one</remarks>
371
        public Stream GetObject(string container, string objectName)
372
        {
373
            if (String.IsNullOrWhiteSpace(container))
374
                throw new ArgumentNullException("container", "The container property can't be empty");
375
            if (String.IsNullOrWhiteSpace(objectName))
376
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
377

    
378
            var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Get };
379
/*
380
            if (DownloadPercentLimit > 0)
381
                request.TaskOptions = new TaskOptions<int> { RateLimitPercent = DownloadPercentLimit };
382
*/
383
            
384
            var response = _client.Request(request);
385
            
386
            if (response.StatusCode == HttpStatusCode.NotFound)
387
                throw new FileNotFoundException();
388
            if (response.StatusCode == HttpStatusCode.OK)
389
            {
390
                return response.ContentStream;
391
            }
392
            else
393
                throw new WebException(String.Format("GetObject failed with unexpected status code {0}", response.StatusCode));
394
        }
395

    
396
        /// <summary>
397
        /// 
398
        /// </summary>
399
        /// <param name="container"></param>
400
        /// <param name="objectName"></param>
401
        /// <param name="fileName"></param>
402
        /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
403
        /// <remarks>>This method should have no timeout or a very long one</remarks>
404
        public Task PutObject(string container, string objectName, string fileName, string hash = null)
405
        {
406
            if (String.IsNullOrWhiteSpace(container))
407
                throw new ArgumentNullException("container", "The container property can't be empty");
408
            if (String.IsNullOrWhiteSpace(objectName))
409
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
410
            if (String.IsNullOrWhiteSpace(fileName))
411
                throw new ArgumentNullException("fileName", "The fileName property can't be empty");
412
            if (!File.Exists(fileName))
413
                throw new FileNotFoundException("The file does not exist",fileName);
414

    
415
            
416
            try
417
            {
418
                var url = String.Join("/",new[]{_client.Authority,container,objectName});
419
                var uri = new Uri(url);
420

    
421
                var client = new WebClient();                
422
                string etag = hash ?? CalculateHash(fileName);
423

    
424
                client.Headers.Add("Content-Type", "application/octet-stream");
425
                client.Headers.Add("ETag", etag);
426

    
427
                if(!String.IsNullOrWhiteSpace(_client.Proxy))
428
                    client.Proxy = new WebProxy(_client.Proxy);
429

    
430
                CopyHeaders(_client, client);
431

    
432
                Trace.TraceInformation("[PUT] START {0}", objectName);
433
                client.UploadProgressChanged += (sender, args) =>
434
                {
435
                    Trace.TraceInformation("[PROGRESS] {0} {1}% {2} of {3}", fileName, args.ProgressPercentage, args.BytesSent, args.TotalBytesToSend);
436
                };
437
               
438
                return client.UploadFileTask(uri, "PUT", fileName)
439
                    .ContinueWith(upload=>
440
                                      {
441
                                          client.Dispose();
442

    
443
                                          if (upload.IsFaulted)
444
                                          {                                              
445
                                              Trace.TraceError("[PUT] FAIL for {0} with \r{1}",objectName,upload.Exception);
446
                                          }
447
                                          else
448
                                            Trace.TraceInformation("[PUT] END {0}", objectName);
449
                                      });
450
            }
451
            catch (Exception exc)
452
            {
453
                Trace.TraceError("[PUT] END {0} with {1}", objectName, exc);
454
                throw;
455
            }                
456

    
457
        }
458

    
459
        /// <summary>
460
        /// Copies headers from a Hammock RestClient to a WebClient
461
        /// </summary>
462
        /// <param name="source">The RestClient from which the headers are copied</param>
463
        /// <param name="target">The WebClient to which the headers are copied</param>
464
        private static void CopyHeaders(RestClient source, WebClient target)
465
        {
466
            Contract.Requires(source!=null,"source can't be null");
467
            Contract.Requires(target != null, "target can't be null");
468
            if (source == null)
469
                throw new ArgumentNullException("source", "source can't be null");
470
            if (source == null)
471
                throw new ArgumentNullException("target", "target can't be null");
472

    
473
            foreach (var header in source.GetAllHeaders())
474
            {
475
                target.Headers.Add(header.Name, header.Value);
476
            }
477
        }
478

    
479
        private static string CalculateHash(string fileName)
480
        {
481
            string hash;
482
            using (var hasher = MD5.Create())
483
            using(var stream=File.OpenRead(fileName))
484
            {
485
                var hashBuilder=new StringBuilder();
486
                foreach (byte b in hasher.ComputeHash(stream))
487
                    hashBuilder.Append(b.ToString("x2").ToLower());
488
                hash = hashBuilder.ToString();                
489
            }
490
            return hash;
491
        }
492

    
493
        public void DeleteObject(string container, string objectName)
494
        {
495
            if (String.IsNullOrWhiteSpace(container))
496
                throw new ArgumentNullException("container", "The container property can't be empty");
497
            if (String.IsNullOrWhiteSpace(objectName))
498
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
499

    
500
            var request = new RestRequest { Path = container + "/" + objectName, Method = WebMethod.Delete, RetryPolicy = _retryPolicy,Timeout = _shortTimeout };
501
            var response = _client.Request(request);
502

    
503
            if (response.StatusCode == HttpStatusCode.NotFound || response.StatusCode == HttpStatusCode.NoContent)
504
                return;
505
            else
506
                throw new WebException(String.Format("DeleteObject failed with unexpected status code {0}", response.StatusCode));
507
   
508
        }
509

    
510
        public void MoveObject(string container, string oldObjectName, string newObjectName)
511
        {
512
            if (String.IsNullOrWhiteSpace(container))
513
                throw new ArgumentNullException("container", "The container property can't be empty");
514
            if (String.IsNullOrWhiteSpace(oldObjectName))
515
                throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
516
            if (String.IsNullOrWhiteSpace(newObjectName))
517
                throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
518

    
519
            var request = new RestRequest { Path = container + "/" + newObjectName, Method = WebMethod.Put };
520
            request.AddHeader("X-Copy-From",String.Format("/{0}/{1}",container,oldObjectName));
521
            request.AddPostContent(new byte[]{});
522
            var response = _client.Request(request);
523

    
524
            if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.NoContent || response.StatusCode==HttpStatusCode.Created)
525
            {
526
                this.DeleteObject(container,oldObjectName);
527
            }                
528
            else
529
                throw new WebException(String.Format("MoveObject failed with unexpected status code {0}", response.StatusCode));
530
        }
531

    
532
        private string GetHeaderValue(string headerName, RestResponse response, IQueryable<string> keys)
533
        {
534
            if (keys.Any(key => key == headerName))
535
                return response.Headers[headerName];
536
            else
537
                throw new WebException(String.Format("The {0}  header is missing",headerName));
538
        }
539

    
540
        private static void ThrowIfNotStatusOK(RestResponse response, string message)
541
        {
542
            int status = (int)response.StatusCode;
543
            if (status < 200 || status >= 300)
544
                throw new WebException(String.Format("{0} with code {1}",message, status));
545
        }
546
    }
547
}