Statistics
| Branch: | Revision:

root / trunk / Pithos.Network / CloudFilesClient.cs @ 7b0a5fec

History | View | Annotate | Download (48.6 kB)

1
// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
2
//
3
// The class provides methods to upload/download files, delete files, manage containers
4

    
5

    
6
using System;
7
using System.Collections.Generic;
8
using System.ComponentModel.Composition;
9
using System.Diagnostics.Contracts;
10
using System.IO;
11
using System.Linq;
12
using System.Net;
13
using System.Security.Cryptography;
14
using System.Text;
15
using System.Threading.Tasks;
16
using Newtonsoft.Json;
17
using Pithos.Interfaces;
18
using log4net;
19

    
20
namespace Pithos.Network
21
{
22
    [Export(typeof(ICloudClient))]
23
    public class CloudFilesClient:ICloudClient
24
    {
25
        //CloudFilesClient uses *_baseClient* internally to communicate with the server
26
        //RestClient provides a REST-friendly interface over the standard WebClient.
27
        private RestClient _baseClient;
28
        
29

    
30
        //During authentication the client provides a UserName 
31
        public string UserName { get; set; }
32
        
33
        //and and ApiKey to the server
34
        public string ApiKey { get; set; }
35
        
36
        //And receives an authentication Token. This token must be provided in ALL other operations,
37
        //in the X-Auth-Token header
38
        private string _token;
39
        public string Token
40
        {
41
            get { return _token; }
42
            set
43
            {
44
                _token = value;
45
                _baseClient.Headers["X-Auth-Token"] = value;
46
            }
47
        }
48

    
49
        //The client also receives a StorageUrl after authentication. All subsequent operations must
50
        //use this url
51
        public Uri StorageUrl { get; set; }
52

    
53

    
54
        protected Uri RootAddressUri { get; set; }
55

    
56
        private Uri _proxy;
57
        public Uri Proxy
58
        {
59
            get { return _proxy; }
60
            set
61
            {
62
                _proxy = value;
63
                if (_baseClient != null)
64
                    _baseClient.Proxy = new WebProxy(value);                
65
            }
66
        }
67

    
68
        public double DownloadPercentLimit { get; set; }
69
        public double UploadPercentLimit { get; set; }
70

    
71
        public string AuthenticationUrl { get; set; }
72

    
73
 
74
        public string VersionPath
75
        {
76
            get { return UsePithos ? "v1" : "v1.0"; }
77
        }
78

    
79
        public bool UsePithos { get; set; }
80

    
81

    
82
        private static readonly ILog Log = LogManager.GetLogger("CloudFilesClient");
83

    
84
        public CloudFilesClient(string userName, string apiKey)
85
        {
86
            UserName = userName;
87
            ApiKey = apiKey;
88
        }
89

    
90
        public CloudFilesClient(AccountInfo accountInfo)
91
        {
92
            if (accountInfo==null)
93
                throw new ArgumentNullException("accountInfo");
94
            Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
95
            Contract.Ensures(StorageUrl != null);
96
            Contract.Ensures(_baseClient != null);
97
            Contract.Ensures(RootAddressUri != null);
98
            Contract.EndContractBlock();
99

    
100
            _baseClient = new RestClient
101
            {
102
                BaseAddress = accountInfo.StorageUri.ToString(),
103
                Timeout = 10000,
104
                Retries = 3
105
            };
106
            StorageUrl = accountInfo.StorageUri;
107
            Token = accountInfo.Token;
108
            UserName = accountInfo.UserName;
109

    
110
            //Get the root address (StorageUrl without the account)
111
            var storageUrl = StorageUrl.AbsoluteUri;
112
            var usernameIndex = storageUrl.LastIndexOf(UserName);
113
            var rootUrl = storageUrl.Substring(0, usernameIndex);
114
            RootAddressUri = new Uri(rootUrl);
115
        }
116

    
117

    
118
        public AccountInfo Authenticate()
119
        {
120
            if (String.IsNullOrWhiteSpace(UserName))
121
                throw new InvalidOperationException("UserName is empty");
122
            if (String.IsNullOrWhiteSpace(ApiKey))
123
                throw new InvalidOperationException("ApiKey is empty");
124
            if (String.IsNullOrWhiteSpace(AuthenticationUrl))
125
                throw new InvalidOperationException("AuthenticationUrl is empty");
126
            Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
127
            Contract.Ensures(StorageUrl != null);
128
            Contract.Ensures(_baseClient != null);
129
            Contract.Ensures(RootAddressUri != null);
130
            Contract.EndContractBlock();
131

    
132

    
133
            Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
134

    
135
            using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
136
            {
137
                if (Proxy != null)
138
                    authClient.Proxy = new WebProxy(Proxy);
139

    
140
                Contract.Assume(authClient.Headers!=null);
141

    
142
                authClient.Headers.Add("X-Auth-User", UserName);
143
                authClient.Headers.Add("X-Auth-Key", ApiKey);
144

    
145
                authClient.DownloadStringWithRetry(VersionPath, 3);
146

    
147
                authClient.AssertStatusOK("Authentication failed");
148

    
149
                var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
150
                if (String.IsNullOrWhiteSpace(storageUrl))
151
                    throw new InvalidOperationException("Failed to obtain storage url");
152
                
153
                _baseClient = new RestClient
154
                {
155
                    BaseAddress = storageUrl,
156
                    Timeout = 10000,
157
                    Retries = 3
158
                };
159

    
160
                StorageUrl = new Uri(storageUrl);
161
                
162
                //Get the root address (StorageUrl without the account)
163
                var usernameIndex=storageUrl.LastIndexOf(UserName);
164
                var rootUrl = storageUrl.Substring(0, usernameIndex);
165
                RootAddressUri = new Uri(rootUrl);
166
                
167
                var token = authClient.GetHeaderValue("X-Auth-Token");
168
                if (String.IsNullOrWhiteSpace(token))
169
                    throw new InvalidOperationException("Failed to obtain token url");
170
                Token = token;
171

    
172
            }
173

    
174
            Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
175
            
176

    
177
            return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName};            
178

    
179
        }
180

    
181

    
182

    
183
        public IList<ContainerInfo> ListContainers(string account)
184
        {
185
            using (var client = new RestClient(_baseClient))
186
            {
187
                if (!String.IsNullOrWhiteSpace(account))
188
                    client.BaseAddress = GetAccountUrl(account);
189
                
190
                client.Parameters.Clear();
191
                client.Parameters.Add("format", "json");
192
                var content = client.DownloadStringWithRetry("", 3);
193
                client.AssertStatusOK("List Containers failed");
194

    
195
                if (client.StatusCode == HttpStatusCode.NoContent)
196
                    return new List<ContainerInfo>();
197
                var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
198
                
199
                foreach (var info in infos)
200
                {
201
                    info.Account = account;
202
                }
203
                return infos;
204
            }
205

    
206
        }
207

    
208
        private string GetAccountUrl(string account)
209
        {
210
            return new Uri(this.RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
211
        }
212

    
213
        public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
214
        {
215
            using (log4net.ThreadContext.Stacks["Share"].Push("List Accounts"))
216
            {
217
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
218

    
219
                using (var client = new RestClient(_baseClient))
220
                {
221
                    client.Parameters.Clear();
222
                    client.Parameters.Add("format", "json");
223
                    client.IfModifiedSince = since;
224

    
225
                    //Extract the username from the base address
226
                    client.BaseAddress = RootAddressUri.AbsoluteUri;
227

    
228
                    var content = client.DownloadStringWithRetry(@"", 3);
229

    
230
                    client.AssertStatusOK("ListSharingAccounts failed");
231

    
232
                    //If the result is empty, return an empty list,
233
                    var infos = String.IsNullOrWhiteSpace(content)
234
                                    ? new List<ShareAccountInfo>()
235
                                //Otherwise deserialize the account list into a list of ShareAccountInfos
236
                                    : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);
237

    
238
                    Log.DebugFormat("END");
239
                    return infos;
240
                }
241
            }
242
        }
243

    
244
        //Request listing of all objects in a container modified since a specific time.
245
        //If the *since* value is missing, return all objects
246
        public IList<ObjectInfo> ListSharedObjects(DateTime? since = null)
247
        {
248

    
249
            using (log4net.ThreadContext.Stacks["Share"].Push("List Objects"))
250
            {
251
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
252

    
253
                var objects = new List<ObjectInfo>();
254
                var accounts = ListSharingAccounts(since);
255
                foreach (var account in accounts)
256
                {
257
                    var containers = ListContainers(account.name);
258
                    foreach (var container in containers)
259
                    {
260
                        var containerObjects = ListObjects(account.name, container.Name, null);
261
                        objects.AddRange(containerObjects);
262
                    }
263
                }
264
                if (Log.IsDebugEnabled) Log.DebugFormat("END");
265
                return objects;
266
            }
267
        }
268

    
269
        public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
270
        {
271
            if (String.IsNullOrWhiteSpace(Token))
272
                throw new InvalidOperationException("The Token is not set");
273
            if (StorageUrl==null)
274
                throw new InvalidOperationException("The StorageUrl is not set");
275
            if (String.IsNullOrWhiteSpace(container))
276
                throw new ArgumentNullException("container");
277
            if (String.IsNullOrWhiteSpace(objectName))
278
                throw new ArgumentNullException("objectName");
279
            if (String.IsNullOrWhiteSpace(account))
280
                throw new ArgumentNullException("account");
281
            if (String.IsNullOrWhiteSpace(shareTo))
282
                throw new ArgumentNullException("shareTo");
283
            Contract.EndContractBlock();
284

    
285
            using (log4net.ThreadContext.Stacks["Share"].Push("Share Object"))
286
            {
287
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
288
                
289
                using (var client = new RestClient(_baseClient))
290
                {
291

    
292
                    client.BaseAddress = GetAccountUrl(account);
293

    
294
                    client.Parameters.Clear();
295
                    client.Parameters.Add("format", "json");
296

    
297
                    string permission = "";
298
                    if (write)
299
                        permission = String.Format("write={0}", shareTo);
300
                    else if (read)
301
                        permission = String.Format("read={0}", shareTo);
302
                    client.Headers.Add("X-Object-Sharing", permission);
303

    
304
                    var content = client.DownloadStringWithRetry(container, 3);
305

    
306
                    client.AssertStatusOK("ShareObject failed");
307

    
308
                    //If the result is empty, return an empty list,
309
                    var infos = String.IsNullOrWhiteSpace(content)
310
                                    ? new List<ObjectInfo>()
311
                                //Otherwise deserialize the object list into a list of ObjectInfos
312
                                    : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
313

    
314
                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
315
                }
316
            }
317

    
318

    
319
        }
320

    
321
        public AccountInfo GetAccountPolicies(AccountInfo accountInfo)
322
        {
323
            if (accountInfo==null)
324
                throw new ArgumentNullException("accountInfo");
325
            Contract.EndContractBlock();
326

    
327
            using (log4net.ThreadContext.Stacks["Account"].Push("GetPolicies"))
328
            {
329
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
330

    
331
                using (var client = new RestClient(_baseClient))
332
                {
333
                    if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
334
                        client.BaseAddress = GetAccountUrl(accountInfo.UserName);
335

    
336
                    client.Parameters.Clear();
337
                    client.Parameters.Add("format", "json");                    
338
                    client.Head(String.Empty, 3);
339

    
340
                    var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
341
                    var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
342

    
343
                    long quota, bytes;
344
                    if (long.TryParse(quotaValue, out quota))
345
                        accountInfo.Quota = quota;
346
                    if (long.TryParse(bytesValue, out bytes))
347
                        accountInfo.BytesUsed = bytes;
348
                    
349
                    return accountInfo;
350

    
351
                }
352

    
353
            }
354
        }
355

    
356

    
357
        public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)
358
        {
359
            if (String.IsNullOrWhiteSpace(container))
360
                throw new ArgumentNullException("container");
361
            Contract.EndContractBlock();
362

    
363
            using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
364
            {
365
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
366

    
367
                using (var client = new RestClient(_baseClient))
368
                {
369
                    if (!String.IsNullOrWhiteSpace(account))
370
                        client.BaseAddress = GetAccountUrl(account);
371

    
372
                    client.Parameters.Clear();
373
                    client.Parameters.Add("format", "json");
374
                    client.IfModifiedSince = since;
375
                    var content = client.DownloadStringWithRetry(container, 3);
376

    
377
                    client.AssertStatusOK("ListObjects failed");
378

    
379
                    //If the result is empty, return an empty list,
380
                    var infos = String.IsNullOrWhiteSpace(content)
381
                                    ? new List<ObjectInfo>()
382
                                //Otherwise deserialize the object list into a list of ObjectInfos
383
                                    : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
384

    
385
                    foreach (var info in infos)
386
                    {
387
                        info.Container = container;
388
                        info.Account = account;
389
                    }
390
                    if (Log.IsDebugEnabled) Log.DebugFormat("START");
391
                    return infos;
392
                }
393
            }
394
        }
395

    
396

    
397

    
398
        public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)
399
        {
400
            if (String.IsNullOrWhiteSpace(container))
401
                throw new ArgumentNullException("container");
402
            if (String.IsNullOrWhiteSpace(folder))
403
                throw new ArgumentNullException("folder");
404
            Contract.EndContractBlock();
405

    
406
            using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
407
            {
408
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
409

    
410
                using (var client = new RestClient(_baseClient))
411
                {
412
                    if (!String.IsNullOrWhiteSpace(account))
413
                        client.BaseAddress = GetAccountUrl(account);
414

    
415
                    client.Parameters.Clear();
416
                    client.Parameters.Add("format", "json");
417
                    client.Parameters.Add("path", folder);
418
                    client.IfModifiedSince = since;
419
                    var content = client.DownloadStringWithRetry(container, 3);
420
                    client.AssertStatusOK("ListObjects failed");
421

    
422
                    var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
423

    
424
                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
425
                    return infos;
426
                }
427
            }
428
        }
429

    
430
 
431
        public bool ContainerExists(string account, string container)
432
        {
433
            if (String.IsNullOrWhiteSpace(container))
434
                throw new ArgumentNullException("container", "The container property can't be empty");
435
            Contract.EndContractBlock();
436

    
437
            using (log4net.ThreadContext.Stacks["Containters"].Push("Exists"))
438
            {
439
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
440

    
441
                using (var client = new RestClient(_baseClient))
442
                {
443
                    if (!String.IsNullOrWhiteSpace(account))
444
                        client.BaseAddress = GetAccountUrl(account);
445

    
446
                    client.Parameters.Clear();
447
                    client.Head(container, 3);
448
                                        
449
                    bool result;
450
                    switch (client.StatusCode)
451
                    {
452
                        case HttpStatusCode.OK:
453
                        case HttpStatusCode.NoContent:
454
                            result=true;
455
                            break;
456
                        case HttpStatusCode.NotFound:
457
                            result=false;
458
                            break;
459
                        default:
460
                            throw CreateWebException("ContainerExists", client.StatusCode);
461
                    }
462
                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
463

    
464
                    return result;
465
                }
466
                
467
            }
468
        }
469

    
470
        public bool ObjectExists(string account, string container, string objectName)
471
        {
472
            if (String.IsNullOrWhiteSpace(container))
473
                throw new ArgumentNullException("container", "The container property can't be empty");
474
            if (String.IsNullOrWhiteSpace(objectName))
475
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
476
            Contract.EndContractBlock();
477

    
478
            using (var client = new RestClient(_baseClient))
479
            {
480
                if (!String.IsNullOrWhiteSpace(account))
481
                    client.BaseAddress = GetAccountUrl(account);
482

    
483
                client.Parameters.Clear();
484
                client.Head(container + "/" + objectName, 3);
485

    
486
                switch (client.StatusCode)
487
                {
488
                    case HttpStatusCode.OK:
489
                    case HttpStatusCode.NoContent:
490
                        return true;
491
                    case HttpStatusCode.NotFound:
492
                        return false;
493
                    default:
494
                        throw CreateWebException("ObjectExists", client.StatusCode);
495
                }
496
            }
497

    
498
        }
499

    
500
        public ObjectInfo GetObjectInfo(string account, string container, string objectName)
501
        {
502
            if (String.IsNullOrWhiteSpace(container))
503
                throw new ArgumentNullException("container", "The container property can't be empty");
504
            if (String.IsNullOrWhiteSpace(objectName))
505
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
506
            Contract.EndContractBlock();
507

    
508
            using (log4net.ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
509
            {                
510

    
511
                using (var client = new RestClient(_baseClient))
512
                {
513
                    if (!String.IsNullOrWhiteSpace(account))
514
                        client.BaseAddress = GetAccountUrl(account);
515
                    try
516
                    {
517
                        client.Parameters.Clear();
518

    
519
                        client.Head(container + "/" + objectName, 3);
520

    
521
                        if (client.TimedOut)
522
                            return ObjectInfo.Empty;
523

    
524
                        switch (client.StatusCode)
525
                        {
526
                            case HttpStatusCode.OK:
527
                            case HttpStatusCode.NoContent:
528
                                var keys = client.ResponseHeaders.AllKeys.AsQueryable();
529
                                var tags = (from key in keys
530
                                            where key.StartsWith("X-Object-Meta-")
531
                                            let name = key.Substring(14)
532
                                            select new {Name = name, Value = client.ResponseHeaders[name]})
533
                                    .ToDictionary(t => t.Name, t => t.Value);
534
                                var extensions = (from key in keys
535
                                                  where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
536
                                                  select new {Name = key, Value = client.ResponseHeaders[key]})
537
                                    .ToDictionary(t => t.Name, t => t.Value);
538
                                var info = new ObjectInfo
539
                                               {
540
                                                   Account = account,
541
                                                   Container = container,
542
                                                   Name = objectName,
543
                                                   Hash = client.GetHeaderValue("ETag"),
544
                                                   Content_Type = client.GetHeaderValue("Content-Type"),
545
                                                   Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length")),
546
                                                   Tags = tags,
547
                                                   Last_Modified = client.LastModified,
548
                                                   Extensions = extensions
549
                                               };
550
                                return info;
551
                            case HttpStatusCode.NotFound:
552
                                return ObjectInfo.Empty;
553
                            default:
554
                                throw new WebException(
555
                                    String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
556
                                                  objectName, client.StatusCode));
557
                        }
558

    
559
                    }
560
                    catch (RetryException)
561
                    {
562
                        Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.");
563
                        return ObjectInfo.Empty;
564
                    }
565
                    catch (WebException e)
566
                    {
567
                        Log.Error(
568
                            String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
569
                                          objectName, client.StatusCode), e);
570
                        throw;
571
                    }
572
                }                
573
            }
574

    
575
        }
576

    
577
        public void CreateFolder(string account, string container, string folder)
578
        {
579
            if (String.IsNullOrWhiteSpace(container))
580
                throw new ArgumentNullException("container", "The container property can't be empty");
581
            if (String.IsNullOrWhiteSpace(folder))
582
                throw new ArgumentNullException("folder", "The folder property can't be empty");
583
            Contract.EndContractBlock();
584

    
585
            var folderUrl=String.Format("{0}/{1}",container,folder);
586
            using (var client = new RestClient(_baseClient))
587
            {
588
                if (!String.IsNullOrWhiteSpace(account))
589
                    client.BaseAddress = GetAccountUrl(account);
590

    
591
                client.Parameters.Clear();
592
                client.Headers.Add("Content-Type", @"application/directory");
593
                client.Headers.Add("Content-Length", "0");
594
                client.PutWithRetry(folderUrl, 3);
595

    
596
                if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
597
                    throw CreateWebException("CreateFolder", client.StatusCode);
598
            }
599
        }
600

    
601
        public ContainerInfo GetContainerInfo(string account, string container)
602
        {
603
            if (String.IsNullOrWhiteSpace(container))
604
                throw new ArgumentNullException("container", "The container property can't be empty");
605
            Contract.EndContractBlock();
606

    
607
            using (var client = new RestClient(_baseClient))
608
            {
609
                if (!String.IsNullOrWhiteSpace(account))
610
                    client.BaseAddress = GetAccountUrl(account);                
611

    
612
                client.Head(container);
613
                switch (client.StatusCode)
614
                {
615
                    case HttpStatusCode.OK:
616
                    case HttpStatusCode.NoContent:
617
                        var containerInfo = new ContainerInfo
618
                                                {
619
                                                    Name = container,
620
                                                    Count =
621
                                                        long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
622
                                                    Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
623
                                                    BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
624
                                                    BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size"))
625
                                                };
626
                        return containerInfo;
627
                    case HttpStatusCode.NotFound:
628
                        return ContainerInfo.Empty;
629
                    default:
630
                        throw CreateWebException("GetContainerInfo", client.StatusCode);
631
                }
632
            }
633
        }
634

    
635
        public void CreateContainer(string account, string container)
636
        {            
637
            if (String.IsNullOrWhiteSpace(container))
638
                throw new ArgumentNullException("container", "The container property can't be empty");
639
            Contract.EndContractBlock();
640

    
641
            using (var client = new RestClient(_baseClient))
642
            {
643
                if (!String.IsNullOrWhiteSpace(account))
644
                    client.BaseAddress = GetAccountUrl(account);
645

    
646
                client.PutWithRetry(container, 3);
647
                var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
648
                if (!expectedCodes.Contains(client.StatusCode))
649
                    throw CreateWebException("CreateContainer", client.StatusCode);
650
            }
651
        }
652

    
653
        public void DeleteContainer(string account, string container)
654
        {
655
            if (String.IsNullOrWhiteSpace(container))
656
                throw new ArgumentNullException("container", "The container property can't be empty");
657
            Contract.EndContractBlock();
658

    
659
            using (var client = new RestClient(_baseClient))
660
            {
661
                if (!String.IsNullOrWhiteSpace(account))
662
                    client.BaseAddress = GetAccountUrl(account);
663

    
664
                client.DeleteWithRetry(container, 3);
665
                var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
666
                if (!expectedCodes.Contains(client.StatusCode))
667
                    throw CreateWebException("DeleteContainer", client.StatusCode);
668
            }
669

    
670
        }
671

    
672
        /// <summary>
673
        /// 
674
        /// </summary>
675
        /// <param name="account"></param>
676
        /// <param name="container"></param>
677
        /// <param name="objectName"></param>
678
        /// <param name="fileName"></param>
679
        /// <returns></returns>
680
        /// <remarks>This method should have no timeout or a very long one</remarks>
681
        //Asynchronously download the object specified by *objectName* in a specific *container* to 
682
        // a local file
683
        public Task GetObject(string account, string container, string objectName, string fileName)
684
        {
685
            if (String.IsNullOrWhiteSpace(container))
686
                throw new ArgumentNullException("container", "The container property can't be empty");
687
            if (String.IsNullOrWhiteSpace(objectName))
688
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");            
689
            Contract.EndContractBlock();
690

    
691
            try
692
            {
693
                //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
694
                //object to avoid concurrency errors.
695
                //
696
                //Download operations take a long time therefore they have no timeout.
697
                var client = new RestClient(_baseClient) { Timeout = 0 };
698
                if (!String.IsNullOrWhiteSpace(account))
699
                    client.BaseAddress = GetAccountUrl(account);
700

    
701
                //The container and objectName are relative names. They are joined with the client's
702
                //BaseAddress to create the object's absolute address
703
                var builder = client.GetAddressBuilder(container, objectName);
704
                var uri = builder.Uri;
705

    
706
                //Download progress is reported to the Trace log
707
                Log.InfoFormat("[GET] START {0}", objectName);
708
                client.DownloadProgressChanged += (sender, args) => 
709
                    Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
710
                                    fileName, args.ProgressPercentage,
711
                                    args.BytesReceived,
712
                                    args.TotalBytesToReceive);                                
713

    
714

    
715
                //Start downloading the object asynchronously
716
                var downloadTask = client.DownloadFileTask(uri, fileName);
717
                
718
                //Once the download completes
719
                return downloadTask.ContinueWith(download =>
720
                                      {
721
                                          //Delete the local client object
722
                                          client.Dispose();
723
                                          //And report failure or completion
724
                                          if (download.IsFaulted)
725
                                          {
726
                                              Log.ErrorFormat("[GET] FAIL for {0} with \r{1}", objectName,
727
                                                               download.Exception);
728
                                          }
729
                                          else
730
                                          {
731
                                              Log.InfoFormat("[GET] END {0}", objectName);                                             
732
                                          }
733
                                      });
734
            }
735
            catch (Exception exc)
736
            {
737
                Log.ErrorFormat("[GET] END {0} with {1}", objectName, exc);
738
                throw;
739
            }
740

    
741

    
742

    
743
        }
744

    
745
        public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)
746
        {
747
            if (String.IsNullOrWhiteSpace(container))
748
                throw new ArgumentNullException("container");
749
            if (String.IsNullOrWhiteSpace(objectName))
750
                throw new ArgumentNullException("objectName");
751
            if (hash==null)
752
                throw new ArgumentNullException("hash");
753
            if (String.IsNullOrWhiteSpace(Token))
754
                throw new InvalidOperationException("Invalid Token");
755
            if (StorageUrl == null)
756
                throw new InvalidOperationException("Invalid Storage Url");
757
            Contract.EndContractBlock();
758

    
759

    
760
            //Don't use a timeout because putting the hashmap may be a long process
761
            var client = new RestClient(_baseClient) { Timeout = 0 };
762
            if (!String.IsNullOrWhiteSpace(account))
763
                client.BaseAddress = GetAccountUrl(account);
764

    
765
            //The container and objectName are relative names. They are joined with the client's
766
            //BaseAddress to create the object's absolute address
767
            var builder = client.GetAddressBuilder(container, objectName);
768
            builder.Query = "format=json&hashmap";
769
            var uri = builder.Uri;
770

    
771

    
772
            //Send the tree hash as Json to the server            
773
            client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
774
            var uploadTask=client.UploadStringTask(uri, "PUT", hash.ToJson());
775

    
776
            
777
            return uploadTask.ContinueWith(t =>
778
            {
779

    
780
                var empty = (IList<string>)new List<string>();
781
                
782

    
783
                //The server will respond either with 201-created if all blocks were already on the server
784
                if (client.StatusCode == HttpStatusCode.Created)                    
785
                {
786
                    //in which case we return an empty hash list
787
                    return empty;
788
                }
789
                //or with a 409-conflict and return the list of missing parts
790
                //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception                
791
                if (t.IsFaulted)
792
                {
793
                    var ex = t.Exception.InnerException;
794
                    var we = ex as WebException;
795
                    var response = we.Response as HttpWebResponse;
796
                    if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
797
                    {
798
                        //In case of 409 the missing parts will be in the response content                        
799
                        using (var stream = response.GetResponseStream())
800
                        using(var reader=new StreamReader(stream))
801
                        {
802
                            //We need to cleanup the content before returning it because it contains
803
                            //error content after the list of hashes
804
                            var hashes = new List<string>();
805
                            string line=null;
806
                            //All lines up to the first empty line are hashes
807
                            while(!String.IsNullOrWhiteSpace(line=reader.ReadLine()))
808
                            {
809
                                hashes.Add(line);
810
                            }
811

    
812
                            return hashes;
813
                        }                        
814
                    }
815
                    else
816
                        //Any other status code is unexpected and the exception should be rethrown
817
                        throw ex;
818
                    
819
                }
820
                //Any other status code is unexpected but there was no exception. We can probably continue processing
821
                else
822
                {
823
                    Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);                    
824
                }
825
                return empty;
826
            });
827

    
828
        }
829

    
830
        public Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end)
831
        {
832
            if (String.IsNullOrWhiteSpace(Token))
833
                throw new InvalidOperationException("Invalid Token");
834
            if (StorageUrl == null)
835
                throw new InvalidOperationException("Invalid Storage Url");
836
            if (String.IsNullOrWhiteSpace(container))
837
                throw new ArgumentNullException("container");
838
            if (relativeUrl== null)
839
                throw new ArgumentNullException("relativeUrl");
840
            if (end.HasValue && end<0)
841
                throw new ArgumentOutOfRangeException("end");
842
            if (start<0)
843
                throw new ArgumentOutOfRangeException("start");
844
            Contract.EndContractBlock();
845

    
846

    
847
            //Don't use a timeout because putting the hashmap may be a long process
848
            var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end};
849
            if (!String.IsNullOrWhiteSpace(account))
850
                client.BaseAddress = GetAccountUrl(account);
851

    
852
            var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
853
            var uri = builder.Uri;
854

    
855
            return client.DownloadDataTask(uri)
856
                .ContinueWith(t=>
857
                                  {
858
                                      client.Dispose();
859
                                      return t.Result;
860
                                  });
861
        }
862

    
863

    
864
        public Task PostBlock(string account, string container, byte[] block, int offset, int count)
865
        {
866
            if (String.IsNullOrWhiteSpace(container))
867
                throw new ArgumentNullException("container");
868
            if (block == null)
869
                throw new ArgumentNullException("block");
870
            if (offset < 0 || offset >= block.Length)
871
                throw new ArgumentOutOfRangeException("offset");
872
            if (count < 0 || count > block.Length)
873
                throw new ArgumentOutOfRangeException("count");
874
            if (String.IsNullOrWhiteSpace(Token))
875
                throw new InvalidOperationException("Invalid Token");
876
            if (StorageUrl == null)
877
                throw new InvalidOperationException("Invalid Storage Url");                        
878
            Contract.EndContractBlock();
879

    
880
                        
881
            //Don't use a timeout because putting the hashmap may be a long process
882
            var client = new RestClient(_baseClient) { Timeout = 0 };
883
            if (!String.IsNullOrWhiteSpace(account))
884
                client.BaseAddress = GetAccountUrl(account);
885

    
886
            var builder = client.GetAddressBuilder(container, "");
887
            //We are doing an update
888
            builder.Query = "update";
889
            var uri = builder.Uri;
890

    
891
            client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
892

    
893
            Log.InfoFormat("[BLOCK POST] START");
894

    
895
            client.UploadProgressChanged += (sender, args) => 
896
                Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
897
                                    args.ProgressPercentage, args.BytesSent,
898
                                    args.TotalBytesToSend);
899
            client.UploadFileCompleted += (sender, args) => 
900
                Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
901

    
902
            
903
            //Send the block
904
            var uploadTask = client.UploadDataTask(uri, "POST", block)
905
            .ContinueWith(upload =>
906
            {
907
                client.Dispose();
908

    
909
                if (upload.IsFaulted)
910
                {
911
                    var exception = upload.Exception.InnerException;
912
                    Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exception);                        
913
                    throw exception;
914
                }
915
                    
916
                Log.InfoFormat("[BLOCK POST] END");
917
            });
918
            return uploadTask;            
919
        }
920

    
921

    
922
        public Task<TreeHash> GetHashMap(string account, string container, string objectName)
923
        {
924
            if (String.IsNullOrWhiteSpace(container))
925
                throw new ArgumentNullException("container");
926
            if (String.IsNullOrWhiteSpace(objectName))
927
                throw new ArgumentNullException("objectName");
928
            if (String.IsNullOrWhiteSpace(Token))
929
                throw new InvalidOperationException("Invalid Token");
930
            if (StorageUrl == null)
931
                throw new InvalidOperationException("Invalid Storage Url");
932
            Contract.EndContractBlock();
933

    
934
            try
935
            {
936
                //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
937
                //object to avoid concurrency errors.
938
                //
939
                //Download operations take a long time therefore they have no timeout.
940
                //TODO: Do we really? this is a hashmap operation, not a download
941
                var client = new RestClient(_baseClient) { Timeout = 0 };
942
                if (!String.IsNullOrWhiteSpace(account))
943
                    client.BaseAddress = GetAccountUrl(account);
944

    
945

    
946
                //The container and objectName are relative names. They are joined with the client's
947
                //BaseAddress to create the object's absolute address
948
                var builder = client.GetAddressBuilder(container, objectName);
949
                builder.Query = "format=json&hashmap";
950
                var uri = builder.Uri;
951
                
952
                //Start downloading the object asynchronously
953
                var downloadTask = client.DownloadStringTask(uri);
954
                
955
                //Once the download completes
956
                return downloadTask.ContinueWith(download =>
957
                {
958
                    //Delete the local client object
959
                    client.Dispose();
960
                    //And report failure or completion
961
                    if (download.IsFaulted)
962
                    {
963
                        Log.ErrorFormat("[GET HASH] FAIL for {0} with \r{1}", objectName,
964
                                        download.Exception);
965
                        throw download.Exception;
966
                    }
967
                                          
968
                    //The server will return an empty string if the file is empty
969
                    var json = download.Result;
970
                    var treeHash = TreeHash.Parse(json);
971
                    Log.InfoFormat("[GET HASH] END {0}", objectName);                                             
972
                    return treeHash;
973
                });
974
            }
975
            catch (Exception exc)
976
            {
977
                Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
978
                throw;
979
            }
980

    
981

    
982

    
983
        }
984

    
985

    
986
        /// <summary>
987
        /// 
988
        /// </summary>
989
        /// <param name="account"></param>
990
        /// <param name="container"></param>
991
        /// <param name="objectName"></param>
992
        /// <param name="fileName"></param>
993
        /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
994
        /// <remarks>>This method should have no timeout or a very long one</remarks>
995
        public Task PutObject(string account, string container, string objectName, string fileName, string hash = null)
996
        {
997
            if (String.IsNullOrWhiteSpace(container))
998
                throw new ArgumentNullException("container", "The container property can't be empty");
999
            if (String.IsNullOrWhiteSpace(objectName))
1000
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
1001
            if (String.IsNullOrWhiteSpace(fileName))
1002
                throw new ArgumentNullException("fileName", "The fileName property can't be empty");
1003
            if (!File.Exists(fileName))
1004
                throw new FileNotFoundException("The file does not exist",fileName);
1005
            Contract.EndContractBlock();
1006
            
1007
            try
1008
            {
1009

    
1010
                var client = new RestClient(_baseClient){Timeout=0};
1011
                if (!String.IsNullOrWhiteSpace(account))
1012
                    client.BaseAddress = GetAccountUrl(account);
1013

    
1014
                var builder = client.GetAddressBuilder(container, objectName);
1015
                var uri = builder.Uri;
1016

    
1017
                string etag = hash ?? CalculateHash(fileName);
1018

    
1019
                client.Headers.Add("Content-Type", "application/octet-stream");
1020
                client.Headers.Add("ETag", etag);
1021

    
1022

    
1023
                Log.InfoFormat("[PUT] START {0}", objectName);
1024
                client.UploadProgressChanged += (sender, args) =>
1025
                {
1026
                    using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
1027
                    {
1028
                        Log.InfoFormat("{0} {1}% {2} of {3}", fileName, args.ProgressPercentage,
1029
                                       args.BytesSent, args.TotalBytesToSend);
1030
                    }
1031
                };
1032

    
1033
                client.UploadFileCompleted += (sender, args) =>
1034
                {
1035
                    using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
1036
                    {
1037
                        Log.InfoFormat("Completed {0}", fileName);
1038
                    }
1039
                };
1040
                return client.UploadFileTask(uri, "PUT", fileName)
1041
                    .ContinueWith(upload=>
1042
                                      {
1043
                                          client.Dispose();
1044

    
1045
                                          if (upload.IsFaulted)
1046
                                          {
1047
                                              var exc = upload.Exception.InnerException;
1048
                                              Log.ErrorFormat("[PUT] FAIL for {0} with \r{1}",objectName,exc);
1049
                                              throw exc;
1050
                                          }
1051
                                          else
1052
                                            Log.InfoFormat("[PUT] END {0}", objectName);
1053
                                      });
1054
            }
1055
            catch (Exception exc)
1056
            {
1057
                Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
1058
                throw;
1059
            }                
1060

    
1061
        }
1062
       
1063
        
1064
        private static string CalculateHash(string fileName)
1065
        {
1066
            Contract.Requires(!String.IsNullOrWhiteSpace(fileName));
1067
            Contract.EndContractBlock();
1068

    
1069
            string hash;
1070
            using (var hasher = MD5.Create())
1071
            using(var stream=File.OpenRead(fileName))
1072
            {
1073
                var hashBuilder=new StringBuilder();
1074
                foreach (byte b in hasher.ComputeHash(stream))
1075
                    hashBuilder.Append(b.ToString("x2").ToLower());
1076
                hash = hashBuilder.ToString();                
1077
            }
1078
            return hash;
1079
        }
1080
        
1081
        public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
1082
        {
1083
            if (String.IsNullOrWhiteSpace(sourceContainer))
1084
                throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1085
            if (String.IsNullOrWhiteSpace(oldObjectName))
1086
                throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
1087
            if (String.IsNullOrWhiteSpace(targetContainer))
1088
                throw new ArgumentNullException("targetContainer", "The container property can't be empty");
1089
            if (String.IsNullOrWhiteSpace(newObjectName))
1090
                throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
1091
            Contract.EndContractBlock();
1092

    
1093
            var targetUrl = targetContainer + "/" + newObjectName;
1094
            var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
1095

    
1096
            using (var client = new RestClient(_baseClient))
1097
            {
1098
                if (!String.IsNullOrWhiteSpace(account))
1099
                    client.BaseAddress = GetAccountUrl(account);
1100

    
1101
                client.Headers.Add("X-Move-From", sourceUrl);
1102
                client.PutWithRetry(targetUrl, 3);
1103

    
1104
                var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1105
                if (!expectedCodes.Contains(client.StatusCode))
1106
                    throw CreateWebException("MoveObject", client.StatusCode);
1107
            }
1108
        }
1109

    
1110
        public void DeleteObject(string account, string sourceContainer, string objectName)
1111
        {            
1112
            if (String.IsNullOrWhiteSpace(sourceContainer))
1113
                throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1114
            if (String.IsNullOrWhiteSpace(objectName))
1115
                throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
1116
            Contract.EndContractBlock();
1117

    
1118
            var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
1119
            var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
1120

    
1121
            using (var client = new RestClient(_baseClient))
1122
            {
1123
                if (!String.IsNullOrWhiteSpace(account))
1124
                    client.BaseAddress = GetAccountUrl(account);
1125

    
1126
                client.Headers.Add("X-Move-From", sourceUrl);
1127
                client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
1128
                client.PutWithRetry(targetUrl, 3);
1129

    
1130
                var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
1131
                if (!expectedCodes.Contains(client.StatusCode))
1132
                    throw CreateWebException("DeleteObject", client.StatusCode);
1133
            }
1134
        }
1135

    
1136
      
1137
        private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
1138
        {
1139
            return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
1140
        }
1141

    
1142
        
1143
    }
1144

    
1145
    public class ShareAccountInfo
1146
    {
1147
        public DateTime? last_modified { get; set; }
1148
        public string name { get; set; }
1149
    }
1150
}