Statistics
| Branch: | Revision:

root / trunk / Pithos.Network / CloudFilesClient.cs @ 2fdc1973

History | View | Annotate | Download (86.8 kB)

1
#region
2
/* -----------------------------------------------------------------------
3
 * <copyright file="CloudFilesClient.cs" company="GRNet">
4
 * 
5
 * Copyright 2011-2012 GRNET S.A. All rights reserved.
6
 *
7
 * Redistribution and use in source and binary forms, with or
8
 * without modification, are permitted provided that the following
9
 * conditions are met:
10
 *
11
 *   1. Redistributions of source code must retain the above
12
 *      copyright notice, this list of conditions and the following
13
 *      disclaimer.
14
 *
15
 *   2. Redistributions in binary form must reproduce the above
16
 *      copyright notice, this list of conditions and the following
17
 *      disclaimer in the documentation and/or other materials
18
 *      provided with the distribution.
19
 *
20
 *
21
 * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
22
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
23
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
25
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
28
 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32
 * POSSIBILITY OF SUCH DAMAGE.
33
 *
34
 * The views and conclusions contained in the software and
35
 * documentation are those of the authors and should not be
36
 * interpreted as representing official policies, either expressed
37
 * or implied, of GRNET S.A.
38
 * </copyright>
39
 * -----------------------------------------------------------------------
40
 */
41
#endregion
42

    
43
// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
44
//
45
// The class provides methods to upload/download files, delete files, manage containers
46

    
47

    
48
using System;
49
using System.Collections.Generic;
50
using System.Collections.Specialized;
51
using System.ComponentModel.Composition;
52
using System.Diagnostics;
53
using System.Diagnostics.Contracts;
54
using System.IO;
55
using System.Linq;
56
using System.Net;
57
using System.Net.Http;
58
using System.Net.Http.Headers;
59
using System.Reflection;
60
using System.ServiceModel.Channels;
61
using System.Text;
62
using System.Threading;
63
using System.Threading.Tasks;
64
using Newtonsoft.Json;
65
using Newtonsoft.Json.Linq;
66
using Pithos.Interfaces;
67
using Pithos.Network;
68
using log4net;
69

    
70
namespace Pithos.Network
71
{
72

    
73
    [Export(typeof(ICloudClient))]
74
    public class CloudFilesClient:ICloudClient,IDisposable
75
    {
76
        private const string TOKEN_HEADER = "X-Auth-Token";
77
        private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
78

    
79
        //CloudFilesClient uses *_baseClient* internally to communicate with the server
80
        //RestClient provides a REST-friendly interface over the standard WebClient.
81
        private RestClient _baseClient;
82

    
83
        private HttpClient _baseHttpClient;
84
        private HttpClient _baseHttpClientNoTimeout;
85
        
86

    
87
        //During authentication the client provides a UserName 
88
        public string UserName { get; set; }
89
        
90
        //and and ApiKey to the server
91
        public string ApiKey { get; set; }
92
        
93
        //And receives an authentication Token. This token must be provided in ALL other operations,
94
        //in the X-Auth-Token header
95
        private string _token;
96
        private readonly string _emptyGuid = Guid.Empty.ToString();
97
        private readonly Uri _emptyUri = new Uri("",UriKind.Relative);
98

    
99
        private HttpClientHandler _httpClientHandler = new HttpClientHandler
100
                                                           {
101
                                                               AllowAutoRedirect = true,
102
                                                               AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
103
                                                               UseCookies = true,   
104
                                                           };
105

    
106

    
107
        public string Token
108
        {
109
            get { return _token; }
110
            set
111
            {
112
                _token = value;
113
                _baseClient.Headers[TOKEN_HEADER] = value;
114
            }
115
        }
116

    
117
        //The client also receives a StorageUrl after authentication. All subsequent operations must
118
        //use this url
119
        public Uri StorageUrl { get; set; }
120

    
121

    
122
        public Uri RootAddressUri { get; set; }
123

    
124

    
125
        public double DownloadPercentLimit { get; set; }
126
        public double UploadPercentLimit { get; set; }
127

    
128
        public string AuthenticationUrl { get; set; }
129

    
130
 
131
        public string VersionPath
132
        {
133
            get { return UsePithos ? "v1" : "v1.0"; }
134
        }
135

    
136
        public bool UsePithos { get; set; }
137

    
138

    
139
        BufferManager _bufferManager=BufferManager.CreateBufferManager(TreeHash.DEFAULT_BLOCK_SIZE*4,(int)TreeHash.DEFAULT_BLOCK_SIZE);
140
        private string _userCatalogUrl;
141

    
142
        public CloudFilesClient(string userName, string apiKey)
143
        {
144
            UserName = userName;
145
            ApiKey = apiKey;
146
            _userCatalogUrl = "https://pithos.okeanos.io/user_catalogs";
147
        }
148

    
149
        public CloudFilesClient(AccountInfo accountInfo)
150
        {
151
            Contract.Requires<ArgumentNullException>(accountInfo!=null,"accountInfo is null");
152
            Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
153
            Contract.Ensures(StorageUrl != null);
154
            Contract.Ensures(_baseClient != null);
155
            Contract.Ensures(RootAddressUri != null);
156
            Contract.EndContractBlock();          
157

    
158
            _baseClient = new RestClient
159
            {
160
                BaseAddress = accountInfo.StorageUri.ToString(),
161
                Timeout = 30000,
162
                Retries = 3,
163
            };
164
            StorageUrl = accountInfo.StorageUri;
165
            Token = accountInfo.Token;
166
            UserName = accountInfo.UserName;
167
            _userCatalogUrl = "https://pithos.okeanos.io/user_catalogs";
168
            //Get the root address (StorageUrl without the account)
169
            var storageUrl = StorageUrl.AbsoluteUri;
170
            var usernameIndex = storageUrl.LastIndexOf(UserName);
171
            var rootUrl = storageUrl.Substring(0, usernameIndex);
172
            RootAddressUri = new Uri(rootUrl);
173

    
174
            var httpClientHandler = new HttpClientHandler
175
            {
176
                AllowAutoRedirect = true,
177
                AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
178
                UseCookies = true,
179
            };
180

    
181

    
182
            _baseHttpClient = new HttpClient(httpClientHandler)
183
            {
184
                BaseAddress = StorageUrl,
185
                Timeout = TimeSpan.FromSeconds(30)
186
            };
187
            _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);
188

    
189
            _baseHttpClientNoTimeout = new HttpClient(httpClientHandler)
190
            {
191
                BaseAddress = StorageUrl,
192
                Timeout = TimeSpan.FromMilliseconds(-1)
193
            };
194
            _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, Token);
195

    
196

    
197
        }
198

    
199

    
200
        private static void AssertStatusOK(HttpResponseMessage response, string message)
201
        {
202
            var statusCode = response.StatusCode;
203
            if (statusCode >= HttpStatusCode.BadRequest)
204
                throw new WebException(String.Format("{0} with code {1} - {2}", message, statusCode, response.ReasonPhrase));
205
        }
206

    
207
        public async Task<AccountInfo> Authenticate()
208
        {
209
            if (String.IsNullOrWhiteSpace(UserName))
210
                throw new InvalidOperationException("UserName is empty");
211
            if (String.IsNullOrWhiteSpace(ApiKey))
212
                throw new InvalidOperationException("ApiKey is empty");
213
            if (String.IsNullOrWhiteSpace(AuthenticationUrl))
214
                throw new InvalidOperationException("AuthenticationUrl is empty");
215
            //if (String.IsNullOrWhiteSpace(Token))
216
            //    throw new InvalidOperationException("Token is Empty");
217
            
218
            //Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
219
            //Contract.Ensures(StorageUrl != null);
220
            //Contract.Ensures(_baseClient != null);
221
            //Contract.Ensures(RootAddressUri != null);
222
            //Contract.EndContractBlock();
223

    
224

    
225
            Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
226

    
227
            var groups = new List<Group>();
228

    
229
//            using (var authClient = new HttpClient(_httpClientHandler,false){ BaseAddress = new Uri(AuthenticationUrl),Timeout=TimeSpan.FromSeconds(30) })
230
            using (var authClient = new HttpClient(_httpClientHandler, false) { BaseAddress = new Uri(AuthenticationUrl), Timeout = TimeSpan.FromSeconds(30) })
231
            {                
232

    
233
                authClient.DefaultRequestHeaders.Add("X-Auth-User", UserName);
234
                authClient.DefaultRequestHeaders.Add("X-Auth-Key", ApiKey);
235

    
236
                string storageUrl;
237
                string token;
238
                
239
                //using (var response = await authClient.GetAsyncWithRetries(new Uri(VersionPath, UriKind.Relative),3).ConfigureAwait(false)) // .DownloadStringWithRetryRelative(new Uri(VersionPath, UriKind.Relative), 3);                    
240
                using (var response = await authClient.SendAsyncWithRetries(new HttpRequestMessage(HttpMethod.Post,new Uri("", UriKind.Relative)), 3).ConfigureAwait(false))
241
                {
242
                    AssertStatusOK(response,"Authentication failed");
243

    
244
                    var body=await response.Content.ReadAsStringAsync();
245
                    JObject json = (JObject) JsonConvert.DeserializeObject(body);
246
                    dynamic jsonD = json;
247
                    
248
                    storageUrl = json["access"]["serviceCatalog"][6]["endpoints"][0]["publicURL"].ToString();
249
                    if (String.IsNullOrWhiteSpace(storageUrl))
250
                        throw new InvalidOperationException("Failed to obtain storage url");
251

    
252
                    token = this.ApiKey;// response.Headers.GetFirstValue(TOKEN_HEADER);
253
                    if (String.IsNullOrWhiteSpace(token))
254
                        throw new InvalidOperationException("Failed to obtain token url");
255
                }
256

    
257
                _baseClient = new RestClient
258
                {
259
                    BaseAddress = storageUrl,
260
                    Timeout = 30000,
261
                    Retries = 3,                    
262
                };
263

    
264
                StorageUrl = new Uri(storageUrl);
265
                Token = token;
266

    
267
                //Get the root address (StorageUrl without the account)
268
                var usernameIndex=storageUrl.LastIndexOf(UserName);
269
                var rootUrl = storageUrl.Substring(0, usernameIndex);
270
                RootAddressUri = new Uri(rootUrl);
271
                
272

    
273
                _baseHttpClient = new HttpClient(_httpClientHandler,false)
274
                {
275
                    BaseAddress = StorageUrl,
276
                    Timeout = TimeSpan.FromSeconds(30)
277
                };
278
                _baseHttpClient.DefaultRequestHeaders.Add(TOKEN_HEADER, token);
279

    
280
                _baseHttpClientNoTimeout = new HttpClient(_httpClientHandler,false)
281
                {
282
                    BaseAddress = StorageUrl,
283
                    Timeout = TimeSpan.FromMilliseconds(-1)
284
                };
285
                _baseHttpClientNoTimeout.DefaultRequestHeaders.Add(TOKEN_HEADER, token);
286

    
287
                /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();
288
                groups = (from key in keys
289
                            where key.StartsWith("X-Account-Group-")
290
                            let name = key.Substring(16)
291
                            select new Group(name, authClient.ResponseHeaders[key]))
292
                        .ToList();
293
                    
294
*/
295
            }
296

    
297
            Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
298
            Debug.Assert(_baseClient!=null);
299
            var displayName = UserName;
300
            Guid uuid;
301
            if (Guid.TryParse(UserName, out uuid))
302
            {
303
                displayName = await ResolveName(uuid);
304
            }
305
            return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName, DisplayName=displayName, Groups=groups};            
306

    
307
        }
308

    
309
        private static void TraceStart(string method, Uri actualAddress)
310
        {
311
            Log.InfoFormat("[{0}] {1} {2}", method, DateTime.Now, actualAddress);
312
        }
313

    
314
        private async Task<string> GetStringAsync(Uri targetUri, string errorMessage,DateTimeOffset? since=null)
315
        {
316
            TraceStart("GET",targetUri);
317
            var request = new HttpRequestMessage(HttpMethod.Get, targetUri);
318
            //request.Headers.Add("User-Agent", "Pithos+ Custom Header");
319
            if (since.HasValue)
320
            {
321
                request.Headers.IfModifiedSince = since.Value;
322
            }
323
            using (var response = await _baseHttpClient.SendAsyncWithRetries(request,3).ConfigureAwait(false))
324
            {
325
                AssertStatusOK(response, errorMessage);
326

    
327
                if (response.StatusCode == HttpStatusCode.NoContent)
328
                    return String.Empty;
329

    
330
                var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
331
                return content;
332
            }
333
        }
334

    
335
        public async Task<IList<ContainerInfo>> ListContainers(string account)
336
        {
337

    
338
            var targetUrl = GetTargetUrl(account);
339
            var targetUri = new Uri(String.Format("{0}?format=json", targetUrl));
340
            var result = await GetStringAsync(targetUri, "List Containers failed").ConfigureAwait(false);
341
            if (String.IsNullOrWhiteSpace(result))
342
                return new List<ContainerInfo>();
343
            var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(result);
344
            foreach (var info in infos)
345
            {
346
                info.Account = account;
347
            }
348
            return infos;
349
        }
350

    
351
        
352
        private string GetAccountUrl(string account)
353
        {
354
            return RootAddressUri.Combine(account).AbsoluteUri;
355
        }
356

    
357
        public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
358
        {
359
            using (ThreadContext.Stacks["Share"].Push("List Accounts"))
360
            {
361
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
362

    
363
                var targetUri = new Uri(String.Format("{0}?format=json", RootAddressUri), UriKind.Absolute);
364
                var content=TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListSharingAccounts failed", since).ConfigureAwait(false)).Result;
365

    
366
                //If the result is empty, return an empty list,
367
                var infos = String.IsNullOrWhiteSpace(content)
368
                                ? new List<ShareAccountInfo>()
369
                            //Otherwise deserialize the account list into a list of ShareAccountInfos
370
                                : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);
371

    
372
                Log.DebugFormat("END");
373
                return infos;
374
            }
375
        }
376

    
377

    
378
        /// <summary>
379
        /// Request listing of all objects in a container modified since a specific time.
380
        /// If the *since* value is missing, return all objects
381
        /// </summary>
382
        /// <param name="knownContainers">Use the since variable only for the containers listed in knownContainers. Unknown containers are considered new
383
        /// and should be polled anyway
384
        /// </param>
385
        /// <param name="since"></param>
386
        /// <returns></returns>
387
        public IList<ObjectInfo> ListSharedObjects(HashSet<string> knownContainers, DateTimeOffset? since)
388
        {
389

    
390
            using (ThreadContext.Stacks["Share"].Push("List Objects"))
391
            {
392
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
393
                //'since' is not used here because we need to have ListObjects return a NoChange result
394
                //for all shared accounts,containers
395

    
396
                Func<ContainerInfo, string> getKey = c => String.Format("{0}\\{1}", c.Account, c.Name);
397
                
398
                var containers = (from account in ListSharingAccounts()
399
                                 let conts = TaskEx.Run(async ()=>await ListContainers(account.name).ConfigureAwait(false)).Result
400
                                 from container in conts
401
                                 select container).ToList();                
402
                var items = from container in containers 
403
                            let actualSince=knownContainers.Contains(getKey(container))?since:null
404
                            select ListObjects(container.Account , container.Name,  actualSince);
405
                var objects=items.SelectMany(r=> r).ToList();
406

    
407
                //For each object
408
                //Check parents recursively up to (but not including) the container.
409
                //If parents are missing, add them to the list
410
                //Need function to calculate all parent URLs
411
                objects = AddMissingParents(objects);
412
                
413
                //Store any new containers
414
                foreach (var container in containers)
415
                {
416
                    knownContainers.Add(getKey(container));
417
                }
418

    
419
                if (Log.IsDebugEnabled) Log.DebugFormat("END");
420
                return objects;
421
            }
422
        }
423

    
424
        private List<ObjectInfo> AddMissingParents(List<ObjectInfo> objects)
425
        {
426
            //TODO: Remove short-circuit when we decide to use Missing Parents functionality
427
            //return objects;
428

    
429
            var existingUris = objects.ToDictionary(o => o.Uri, o => o);
430
            foreach (var objectInfo in objects)
431
            {
432
                //Can be null when retrieving objects to show in selective sync
433
                if (objectInfo.Name == null)
434
                    continue;
435

    
436
                //No need to unescape here, the parts will be used to create new ObjectInfos
437
                var parts = objectInfo.Name.ToString().Split(new[]{'/'},StringSplitOptions.RemoveEmptyEntries);
438
                //If there is no parent, skip
439
                if (parts.Length == 1)
440
                    continue;
441
                var baseParts = new[]
442
                                  {
443
                                      objectInfo.Uri.Host, objectInfo.Uri.Segments[1].TrimEnd('/'),objectInfo.Account,objectInfo.Container.ToString()
444
                                  };
445
                for (var partIdx = 0; partIdx < parts.Length - 1; partIdx++)
446
                {
447
                    var nameparts = parts.Range(0, partIdx).ToArray();
448
                    var parentName= String.Join("/", nameparts);
449

    
450
                    var parentParts = baseParts.Concat(nameparts);
451
                    var parentUrl = objectInfo.Uri.Scheme+ "://" + String.Join("/", parentParts);
452
                    
453
                    var parentUri = new Uri(parentUrl, UriKind.Absolute);
454

    
455
                    ObjectInfo existingInfo;
456
                    if (!existingUris.TryGetValue(parentUri,out existingInfo))
457
                    {
458
                        var h = parentUrl.GetHashCode();
459
                        var reverse = new string(parentUrl.Reverse().ToArray());
460
                        var rh = reverse.GetHashCode();
461
                        var b1 = BitConverter.GetBytes(h);
462
                        var b2 = BitConverter.GetBytes(rh);
463
                        var g = new Guid(0,0,0,b1.Concat(b2).ToArray());
464
                        
465
                        existingUris[parentUri] = new ObjectInfo
466
                                                      {
467
                                                          Account = objectInfo.Account,
468
                                                          Container = objectInfo.Container,
469
                                                          Content_Type = ObjectInfo.CONTENT_TYPE_DIRECTORY,
470
                                                          ETag = Signature.MERKLE_EMPTY,
471
                                                          X_Object_Hash = Signature.MERKLE_EMPTY,
472
                                                          Name=new Uri(parentName,UriKind.Relative),
473
                                                          StorageUri=objectInfo.StorageUri,
474
                                                          Bytes = 0,
475
                                                          UUID=g.ToString(),                                                          
476
                                                      };
477
                    }
478
                }
479
            }
480
            return existingUris.Values.ToList();
481
        }
482

    
483
        public void SetTags(ObjectInfo target,IDictionary<string,string> tags)
484
        {
485
            Contract.Requires<InvalidOperationException>(!String.IsNullOrWhiteSpace(Token),"The Token is not set");
486
            Contract.Requires<InvalidOperationException>(StorageUrl != null,"The StorageUrl is not set");
487
            Contract.Requires<ArgumentNullException>(target != null,"target is null");
488
            Contract.EndContractBlock();
489

    
490
            using (ThreadContext.Stacks["Share"].Push("Share Object"))
491
            {
492
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
493

    
494
                using (var client = new RestClient(_baseClient))
495
                {
496

    
497
                    client.BaseAddress = GetAccountUrl(target.Account);
498

    
499
                    client.Parameters.Clear();
500
                    client.Parameters.Add("update", "");
501

    
502
                    foreach (var tag in tags)
503
                    {
504
                        var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
505
                        client.Headers.Add(headerTag, tag.Value);
506
                    }
507
                    
508
                    client.DownloadStringWithRetryRelative(target.Container, 3);
509

    
510
                    
511
                    client.AssertStatusOK("SetTags failed");
512
                    //If the status is NOT ACCEPTED we have a problem
513
                    if (client.StatusCode != HttpStatusCode.Accepted)
514
                    {
515
                        Log.Error("Failed to set tags");
516
                        throw new Exception("Failed to set tags");
517
                    }
518

    
519
                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
520
                }
521
            }
522

    
523

    
524
        }
525

    
526
        public void ShareObject(string account, Uri container, Uri objectName, string shareTo, bool read, bool write)
527
        {
528

    
529
            Contract.Requires<InvalidOperationException>(!String.IsNullOrWhiteSpace(Token), "The Token is not set");
530
            Contract.Requires<InvalidOperationException>(StorageUrl != null, "The StorageUrl is not set");
531
            Contract.Requires<ArgumentNullException>(container != null, "container is null");
532
            Contract.Requires<ArgumentException>(!container.IsAbsoluteUri, "container is absolute");
533
            Contract.Requires<ArgumentNullException>(objectName != null, "objectName is null");
534
            Contract.Requires<ArgumentException>(!objectName.IsAbsoluteUri, "objectName  is absolute");
535
            Contract.Requires<ArgumentNullException>(!String.IsNullOrWhiteSpace(account), "account is not set");
536
            Contract.Requires<ArgumentNullException>(!String.IsNullOrWhiteSpace(shareTo), "shareTo is not set");
537
            Contract.EndContractBlock();
538

    
539
            using (ThreadContext.Stacks["Share"].Push("Share Object"))
540
            {
541
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
542
                
543
                using (var client = new RestClient(_baseClient))
544
                {
545

    
546
                    client.BaseAddress = GetAccountUrl(account);
547

    
548
                    client.Parameters.Clear();
549
                    client.Parameters.Add("format", "json");
550

    
551
                    string permission = "";
552
                    if (write)
553
                        permission = String.Format("write={0}", shareTo);
554
                    else if (read)
555
                        permission = String.Format("read={0}", shareTo);
556
                    client.Headers.Add("X-Object-Sharing", permission);
557

    
558
                    var content = client.DownloadStringWithRetryRelative(container, 3);
559

    
560
                    client.AssertStatusOK("ShareObject failed");
561

    
562
                    //If the result is empty, return an empty list,
563
                    var infos = String.IsNullOrWhiteSpace(content)
564
                                    ? new List<ObjectInfo>()
565
                                //Otherwise deserialize the object list into a list of ObjectInfos
566
                                    : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
567

    
568
                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
569
                }
570
            }
571

    
572

    
573
        }
574

    
575
        public async Task<AccountInfo> GetAccountPolicies(AccountInfo accountInfo)
576
        {
577
            if (accountInfo==null)
578
                throw new ArgumentNullException("accountInfo");
579
            Contract.EndContractBlock();
580

    
581
            using (ThreadContext.Stacks["Account"].Push("GetPolicies"))
582
            {
583
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
584

    
585
/*
586
                if (_baseClient == null)
587
                {
588
                    _baseClient = new RestClient
589
                    {
590
                        BaseAddress = accountInfo.StorageUri.ToString(),
591
                        Timeout = 30000,
592
                        Retries = 3,
593
                    };
594
                }
595

    
596
*/                
597
                var containerUri = GetTargetUri(accountInfo.UserName);
598
                var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute);
599
                using(var response=await _baseHttpClient.HeadAsyncWithRetries(targetUri,3).ConfigureAwait(false))
600
                {
601
                    
602
                    var quotaValue=response.Headers.GetFirstValue("X-Account-Policy-Quota");
603
                    var bytesValue = response.Headers.GetFirstValue("X-Account-Bytes-Used");
604
                    long quota, bytes;
605
                    if (long.TryParse(quotaValue, out quota))
606
                        accountInfo.Quota = quota;
607
                    if (long.TryParse(bytesValue, out bytes))
608
                        accountInfo.BytesUsed = bytes;
609

    
610
                    return accountInfo;   
611
                }
612

    
613

    
614
                //using (var client = new RestClient(_baseClient))
615
                //{
616
                //    if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
617
                //        client.BaseAddress = GetAccountUrl(accountInfo.UserName);
618

    
619
                //    client.Parameters.Clear();
620
                //    client.Parameters.Add("format", "json");                    
621
                //    client.Head(_emptyUri, 3);
622

    
623
                //    var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
624
                //    var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
625

    
626
                //    long quota, bytes;
627
                //    if (long.TryParse(quotaValue, out quota))
628
                //        accountInfo.Quota = quota;
629
                //    if (long.TryParse(bytesValue, out bytes))
630
                //        accountInfo.BytesUsed = bytes;
631
                    
632
                //    return accountInfo;
633

    
634
                //}
635

    
636
            }
637
        }
638

    
639
        public void UpdateMetadata(ObjectInfo objectInfo)
640
        {
641
            Contract.Requires<ArgumentNullException>(objectInfo != null,"objectInfo is null");
642
            Contract.EndContractBlock();
643

    
644
            using (ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))
645
            {
646
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
647

    
648

    
649
                using(var client=new RestClient(_baseClient))
650
                {
651

    
652
                    client.BaseAddress = GetAccountUrl(objectInfo.Account);
653
                    
654
                    client.Parameters.Clear();
655
                    
656

    
657
                    //Set Tags
658
                    foreach (var tag in objectInfo.Tags)
659
                    {
660
                        var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
661
                        client.Headers.Add(headerTag, tag.Value);
662
                    }
663

    
664
                    //Set Permissions
665

    
666
                    var permissions=objectInfo.GetPermissionString();
667
                    client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);
668

    
669
                    client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);
670
                    client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);
671
                    client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);
672
                    var isPublic = objectInfo.IsPublic.ToString().ToLower();
673
                    client.Headers.Add("X-Object-Public", isPublic);
674

    
675

    
676
                    var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);
677
                    client.PostWithRetry(new Uri(address,UriKind.Relative),"application/xml");
678
                    
679
                    client.AssertStatusOK("UpdateMetadata failed");
680
                    //If the status is NOT ACCEPTED or OK we have a problem
681
                    if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
682
                    {
683
                        Log.Error("Failed to update metadata");
684
                        throw new Exception("Failed to update metadata");
685
                    }
686

    
687
                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
688
                }
689
            }
690

    
691
        }
692

    
693
        public void UpdateMetadata(ContainerInfo containerInfo)
694
        {
695
            if (containerInfo == null)
696
                throw new ArgumentNullException("containerInfo");
697
            Contract.EndContractBlock();
698

    
699
            using (ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))
700
            {
701
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
702

    
703

    
704
                using(var client=new RestClient(_baseClient))
705
                {
706

    
707
                    client.BaseAddress = GetAccountUrl(containerInfo.Account);
708
                    
709
                    client.Parameters.Clear();
710
                    
711

    
712
                    //Set Tags
713
                    foreach (var tag in containerInfo.Tags)
714
                    {
715
                        var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);
716
                        client.Headers.Add(headerTag, tag.Value);
717
                    }
718

    
719
                    
720
                    //Set Policies
721
                    foreach (var policy in containerInfo.Policies)
722
                    {
723
                        var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);
724
                        client.Headers.Add(headerPolicy, policy.Value);
725
                    }
726

    
727

    
728
                    var uriBuilder = client.GetAddressBuilder(containerInfo.Name,_emptyUri);
729
                    var uri = uriBuilder.Uri;
730

    
731
                    client.UploadValues(uri,new NameValueCollection());
732

    
733

    
734
                    client.AssertStatusOK("UpdateMetadata failed");
735
                    //If the status is NOT ACCEPTED or OK we have a problem
736
                    if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
737
                    {
738
                        Log.Error("Failed to update metadata");
739
                        throw new Exception("Failed to update metadata");
740
                    }
741

    
742
                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
743
                }
744
            }
745

    
746
        }
747

    
748
       
749

    
750

    
751
        public IList<ObjectInfo> ListObjects(string account, Uri container, DateTimeOffset? since = null)
752
        {
753
/*
754
            if (container==null)
755
                throw new ArgumentNullException("container");
756
            if (container.IsAbsoluteUri)
757
                throw new ArgumentException("container");
758
            Contract.EndContractBlock();
759
*/
760

    
761
            using (ThreadContext.Stacks["Objects"].Push("List"))
762
            {
763

    
764
                var containerUri = GetTargetUri(account).Combine(container);
765
                var targetUri = new Uri(String.Format("{0}?format=json", containerUri), UriKind.Absolute);
766

    
767
                var content =TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result;
768

    
769
                //304 will result in an empty string. Empty containers return an empty json array
770
                if (String.IsNullOrWhiteSpace(content))
771
                     return new[] {new NoModificationInfo(account, container)};
772

    
773
                 //If the result is empty, return an empty list,
774
                 var infos = String.IsNullOrWhiteSpace(content)
775
                                 ? new List<ObjectInfo>()
776
                             //Otherwise deserialize the object list into a list of ObjectInfos
777
                                 : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
778

    
779
                 foreach (var info in infos)
780
                 {
781
                     info.Container = container;
782
                     info.Account = account;
783
                     info.StorageUri = StorageUrl;
784
                 }
785
                 if (Log.IsDebugEnabled) Log.DebugFormat("END");
786
                 return infos;
787
            }
788
        }
789

    
790
        public IList<ObjectInfo> ListObjects(string account, Uri container, Uri folder, DateTimeOffset? since = null)
791
        {
792
/*            if (container==null)
793
                throw new ArgumentNullException("container");
794
            if (container.IsAbsoluteUri)
795
                throw new ArgumentException("container");
796
            Contract.EndContractBlock();*/
797

    
798
            using (ThreadContext.Stacks["Objects"].Push("List"))
799
            {
800
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
801

    
802
                var containerUri = GetTargetUri(account).Combine(container);
803
                var targetUri = new Uri(String.Format("{0}?format=json&path={1}", containerUri,folder), UriKind.Absolute);
804
                var content = TaskEx.Run(async ()=>await GetStringAsync(targetUri, "ListObjects failed", since).ConfigureAwait(false)).Result;                
805

    
806
                //304 will result in an empty string. Empty containers return an empty json array
807
                if (String.IsNullOrWhiteSpace(content))
808
                    return new[] { new NoModificationInfo(account, container) };
809

    
810

    
811
                var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
812
                foreach (var info in infos)
813
                {
814
                    info.Account = account;
815
                    if (info.Container == null)
816
                        info.Container = container;
817
                    info.StorageUri = StorageUrl;
818
                }
819
                if (Log.IsDebugEnabled) Log.DebugFormat("END");
820
                return infos;
821
/*
822
                using (var client = new RestClient(_baseClient))
823
                {
824
                    if (!String.IsNullOrWhiteSpace(account))
825
                        client.BaseAddress = GetAccountUrl(account);
826

    
827
                    client.Parameters.Clear();
828
                    client.Parameters.Add("format", "json");
829
                    client.Parameters.Add("path", folder.ToString());
830
                    client.IfModifiedSince = since;
831
                    var content = client.DownloadStringWithRetryRelative(container, 3);
832
                    client.AssertStatusOK("ListObjects failed");
833

    
834
                    if (client.StatusCode==HttpStatusCode.NotModified)
835
                        return new[]{new NoModificationInfo(account,container,folder)};
836

    
837
                    var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
838
                    foreach (var info in infos)
839
                    {
840
                        info.Account = account;
841
                        if (info.Container == null)
842
                            info.Container = container;
843
                        info.StorageUri = StorageUrl;
844
                    }
845
                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
846
                    return infos;
847
                }
848
*/
849
            }
850
        }
851

    
852
 
853
        public async Task<bool> ContainerExists(string account, Uri container)
854
        {
855
            if (container==null)
856
                throw new ArgumentNullException("container", "The container property can't be empty");
857
            if (container.IsAbsoluteUri)
858
                throw new ArgumentException( "The container must be relative","container");
859
            Contract.EndContractBlock();
860

    
861
            using (ThreadContext.Stacks["Containters"].Push("Exists"))
862
            {
863
                if (Log.IsDebugEnabled) Log.DebugFormat("START");
864

    
865
                var targetUri = GetTargetUri(account).Combine(container);
866

    
867
                using (var response =await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3))
868
                {
869

    
870
                    bool result;
871
                    switch (response.StatusCode)
872
                    {
873
                        case HttpStatusCode.OK:
874
                        case HttpStatusCode.NoContent:
875
                            result = true;
876
                            break;
877
                        case HttpStatusCode.NotFound:
878
                            result = false;
879
                            break;
880
                        default:
881
                            throw CreateWebException("ContainerExists", response.StatusCode);
882
                    }
883
                    if (Log.IsDebugEnabled) Log.DebugFormat("END");
884

    
885
                    return result;
886
                }
887

    
888
            }
889
        }
890

    
891
        private Uri GetTargetUri(string account)
892
        {
893
            return new Uri(GetTargetUrl(account),UriKind.Absolute);
894
        }
895

    
896
        private string GetTargetUrl(string account)
897
        {
898
            return String.IsNullOrWhiteSpace(account)
899
                       ? _baseHttpClient.BaseAddress.ToString()
900
                       : GetAccountUrl(account);
901
        }
902

    
903
        public async Task<bool> ObjectExists(string account, Uri container, Uri objectName)
904
        {
905
            if (container == null)
906
                throw new ArgumentNullException("container", "The container property can't be empty");
907
            if (container.IsAbsoluteUri)
908
                throw new ArgumentException("The container must be relative","container");
909
            if (objectName == null)
910
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
911
            if (objectName.IsAbsoluteUri)
912
                throw new ArgumentException("The objectName must be relative","objectName");
913
            Contract.EndContractBlock();
914

    
915
                var targetUri=GetTargetUri(account).Combine(container).Combine(objectName);
916

    
917
            using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3).ConfigureAwait(false))
918
            {
919
                switch (response.StatusCode)
920
                {
921
                    case HttpStatusCode.OK:
922
                    case HttpStatusCode.NoContent:
923
                        return true;
924
                    case HttpStatusCode.NotFound:
925
                        return false;
926
                    default:
927
                        throw CreateWebException("ObjectExists", response.StatusCode);
928
                }
929
            }
930
        }
931

    
932
        public async Task<ObjectInfo> GetObjectInfo(string account, Uri container, Uri objectName)
933
        {
934
            if (container == null)
935
                throw new ArgumentNullException("container", "The container property can't be empty");
936
            if (container.IsAbsoluteUri)
937
                throw new ArgumentException("The container must be relative", "container");
938
            if (objectName == null)
939
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
940
            if (objectName.IsAbsoluteUri)
941
                throw new ArgumentException("The objectName must be relative", "objectName");
942
            Contract.EndContractBlock();
943

    
944
            using (ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
945
            {
946

    
947
                var targetUri = GetTargetUri(account).Combine(container).Combine(objectName);
948
                try
949
                {
950
                    using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3,true))
951
                    {
952
                        switch (response.StatusCode)
953
                        {
954
                            case HttpStatusCode.OK:
955
                            case HttpStatusCode.NoContent:
956
                                var tags = response.Headers.GetMeta("X-Object-Meta-");
957
                                var extensions = (from header in response.Headers
958
                                                  where
959
                                                      header.Key.StartsWith("X-Object-") &&
960
                                                      !header.Key.StartsWith("X-Object-Meta-")
961
                                                  select new {Name = header.Key, Value = header.Value.FirstOrDefault()})
962
                                    .ToDictionary(t => t.Name, t => t.Value);
963

    
964
                                var permissions = response.Headers.GetFirstValue("X-Object-Sharing");
965

    
966

    
967
                                var info = new ObjectInfo
968
                                               {
969
                                                   Account = account,
970
                                                   Container = container,
971
                                                   Name = objectName,
972
                                                   ETag = response.Headers.ETag.NullSafe(e=>e.Tag),
973
                                                   UUID = response.Headers.GetFirstValue("X-Object-UUID"),
974
                                                   X_Object_Hash = response.Headers.GetFirstValue("X-Object-Hash"),
975
                                                   Content_Type = response.Headers.GetFirstValue("Content-Type"),
976
                                                   Bytes = Convert.ToInt64(response.Content.Headers.ContentLength),
977
                                                   Tags = tags,
978
                                                   Last_Modified = response.Content.Headers.LastModified,
979
                                                   Extensions = extensions,
980
                                                   ContentEncoding =
981
                                                       response.Content.Headers.ContentEncoding.FirstOrDefault(),
982
                                                   ContendDisposition =
983
                                                       response.Content.Headers.ContentDisposition.NullSafe(c=>c.ToString()),
984
                                                   Manifest = response.Headers.GetFirstValue("X-Object-Manifest"),
985
                                                   PublicUrl = response.Headers.GetFirstValue("X-Object-Public"),
986
                                                   StorageUri = StorageUrl,
987
                                               };
988
                                info.SetPermissions(permissions);
989
                                return info;
990
                            case HttpStatusCode.NotFound:
991
                                return ObjectInfo.Empty;
992
                            default:
993
                                throw new WebException(
994
                                    String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
995
                                                  objectName, response.StatusCode));
996
                        }
997
                    }
998
                }
999
                catch (RetryException)
1000
                {
1001
                    Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.", objectName);
1002
                    return ObjectInfo.Empty;
1003
                }
1004
                catch (WebException e)
1005
                {
1006
                    Log.Error(
1007
                        String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status {1}",
1008
                                      objectName, e.Status), e);
1009
                    throw;
1010
                }
1011
            }
1012
        }
1013

    
1014

    
1015

    
1016
        public async Task CreateFolder(string account, Uri container, Uri folder)
1017
        {
1018
            if (container == null)
1019
                throw new ArgumentNullException("container", "The container property can't be empty");
1020
            if (container.IsAbsoluteUri)
1021
                throw new ArgumentException("The container must be relative","container");
1022
            if (folder == null)
1023
                throw new ArgumentNullException("folder", "The objectName property can't be empty");
1024
            if (folder.IsAbsoluteUri)
1025
                throw new ArgumentException("The objectName must be relative","folder");
1026
            Contract.EndContractBlock();
1027

    
1028
            var folderUri=container.Combine(folder);            
1029
            var targetUri = GetTargetUri(account).Combine(folderUri);
1030
            var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
1031
            message.Content=new StringContent("");
1032
            message.Content.Headers.ContentType = new MediaTypeHeaderValue(ObjectInfo.CONTENT_TYPE_DIRECTORY);
1033
            //message.Headers.Add("Content-Length", "0");
1034
            using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))
1035
            {
1036
                if (response.StatusCode != HttpStatusCode.Created && response.StatusCode != HttpStatusCode.Accepted)
1037
                    throw CreateWebException("CreateFolder", response.StatusCode);
1038
            }
1039
        }
1040

    
1041
        private Dictionary<string, string> GetMeta(HttpResponseMessage response,string metaPrefix)
1042
        {
1043
            Contract.Requires<ArgumentNullException>(!String.IsNullOrWhiteSpace(metaPrefix), "metaPrefix is empty");
1044
            Contract.EndContractBlock();
1045

    
1046
            var dict = (from header in response.Headers
1047
                        where header.Key.StartsWith(metaPrefix)
1048
                         select new { Name = header.Key, Value = String.Join(",", header.Value) })
1049
                        .ToDictionary(t => t.Name, t => t.Value);
1050

    
1051
          
1052
            return dict;
1053
        }
1054

    
1055

    
1056
        public async Task<ContainerInfo> GetContainerInfo(string account, Uri container)
1057
        {
1058
            if (container == null)
1059
                throw new ArgumentNullException("container", "The container property can't be empty");
1060
            if (container.IsAbsoluteUri)
1061
                throw new ArgumentException("The container must be relative","container");
1062
            Contract.EndContractBlock();
1063

    
1064
            var targetUri = GetTargetUri(account).Combine(container);            
1065
            using (var response = await _baseHttpClient.HeadAsyncWithRetries(targetUri, 3,true).ConfigureAwait(false))
1066
            {
1067
                if (Log.IsDebugEnabled)
1068
                    Log.DebugFormat("ContainerInfo data: {0}\n{1}",response,await response.Content.ReadAsStringAsync().ConfigureAwait(false));
1069
                switch (response.StatusCode)
1070
                {
1071
                    case HttpStatusCode.OK:
1072
                    case HttpStatusCode.NoContent:
1073
                        var tags = GetMeta(response,"X-Container-Meta-");
1074
                        var policies = GetMeta(response,"X-Container-Policy-");
1075

    
1076
                        var containerInfo = new ContainerInfo
1077
                                                {
1078
                                                    Account = account,
1079
                                                    Name = container,
1080
                                                    StorageUrl = StorageUrl.ToString(),
1081
                                                    Count =long.Parse(response.Headers.GetFirstValue("X-Container-Object-Count")),
1082
                                                    Bytes = long.Parse(response.Headers.GetFirstValue("X-Container-Bytes-Used")),
1083
                                                    BlockHash = response.Headers.GetFirstValue("X-Container-Block-Hash"),
1084
                                                    BlockSize =
1085
                                                        int.Parse(response.Headers.GetFirstValue("X-Container-Block-Size")),
1086
                                                    Last_Modified = response.Content.Headers.LastModified,
1087
                                                    Tags = tags,
1088
                                                    Policies = policies
1089
                                                };
1090

    
1091

    
1092
                        return containerInfo;
1093
                    case HttpStatusCode.NotFound:
1094
                        return ContainerInfo.Empty;
1095
                    default:
1096
                        throw CreateWebException("GetContainerInfo", response.StatusCode);
1097
                }
1098
            }            
1099
        }
1100

    
1101
        public async Task CreateContainer(string account, Uri container)
1102
        {
1103
            if (container == null)
1104
                throw new ArgumentNullException("container", "The container property can't be empty");
1105
            if (container.IsAbsoluteUri)
1106
                throw new ArgumentException("The container must be relative","container");
1107
            Contract.EndContractBlock();
1108

    
1109
            var targetUri=GetTargetUri(account).Combine(container);
1110
            var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
1111
            
1112
            //message.Content.Headers.ContentLength = 0;
1113
            using (var response =await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))
1114
            {            
1115
                var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
1116
                if (!expectedCodes.Contains(response.StatusCode))
1117
                    throw CreateWebException("CreateContainer", response.StatusCode);
1118
            }
1119
        }
1120

    
1121
        public async Task WipeContainer(string account, Uri container)
1122
        {
1123
            if (container == null)
1124
                throw new ArgumentNullException("container", "The container property can't be empty");
1125
            if (container.IsAbsoluteUri)
1126
                throw new ArgumentException("The container must be relative", "container");
1127
            Contract.EndContractBlock();
1128

    
1129
            await DeleteContainer(account, new Uri(String.Format("{0}?delimiter=/", container), UriKind.Relative)).ConfigureAwait(false);
1130
        }
1131

    
1132

    
1133
        public async Task DeleteContainer(string account, Uri container)
1134
        {
1135
            if (container == null)
1136
                throw new ArgumentNullException("container", "The container property can't be empty");
1137
            if (container.IsAbsoluteUri)
1138
                throw new ArgumentException("The container must be relative","container");
1139
            Contract.EndContractBlock();
1140

    
1141
            var targetUri = GetTargetUri(account).Combine(container);
1142
            var message = new HttpRequestMessage(HttpMethod.Delete, targetUri);
1143
            using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))
1144
            {
1145
                var expectedCodes = new[] { HttpStatusCode.NotFound, HttpStatusCode.NoContent };
1146
                if (!expectedCodes.Contains(response.StatusCode))
1147
                    throw CreateWebException("DeleteContainer", response.StatusCode);
1148
            }
1149

    
1150
        }
1151

    
1152
        /// <summary>
1153
        /// 
1154
        /// </summary>
1155
        /// <param name="account"></param>
1156
        /// <param name="container"></param>
1157
        /// <param name="objectName"></param>
1158
        /// <param name="fileName"></param>
1159
        /// <param name="cancellationToken"> </param>
1160
        /// <returns></returns>
1161
        /// <remarks>This method should have no timeout or a very long one</remarks>
1162
        //Asynchronously download the object specified by *objectName* in a specific *container* to 
1163
        // a local file
1164
        public async Task GetObject(string account, Uri container, Uri objectName, string fileName,CancellationToken cancellationToken)
1165
        {
1166
            if (container == null)
1167
                throw new ArgumentNullException("container", "The container property can't be empty");
1168
            if (container.IsAbsoluteUri)
1169
                throw new ArgumentException("The container must be relative","container");
1170
            if (objectName == null)
1171
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
1172
            if (objectName.IsAbsoluteUri)
1173
                throw new ArgumentException("The objectName must be relative","objectName");
1174
            Contract.EndContractBlock();
1175
                        
1176

    
1177
            try
1178
            {
1179
                //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
1180
                //object to avoid concurrency errors.
1181
                //
1182
                //Download operations take a long time therefore they have no timeout.
1183
                using(var client = new RestClient(_baseClient) { Timeout = 0 })
1184
                {
1185
                    if (!String.IsNullOrWhiteSpace(account))
1186
                        client.BaseAddress = GetAccountUrl(account);
1187

    
1188
                    //The container and objectName are relative names. They are joined with the client's
1189
                    //BaseAddress to create the object's absolute address
1190
                    var builder = client.GetAddressBuilder(container, objectName);
1191
                    var uri = builder.Uri;
1192

    
1193
                    //Download progress is reported to the Trace log
1194
                    Log.InfoFormat("[GET] START {0}", objectName);
1195
                    /*client.DownloadProgressChanged += (sender, args) =>
1196
                                                      Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
1197
                                                                     fileName, args.ProgressPercentage,
1198
                                                                     args.BytesReceived,
1199
                                                                     args.TotalBytesToReceive);*/
1200
                    var progress = new Progress<DownloadProgressChangedEventArgs>(args =>
1201
                                {
1202
                                    Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
1203
                                                   fileName, args.ProgressPercentage,
1204
                                                   args.BytesReceived,
1205
                                                   args.TotalBytesToReceive);
1206
                                    if (DownloadProgressChanged!=null)
1207
                                        DownloadProgressChanged(this, new DownloadArgs(args));
1208
                                });
1209
                    
1210
                    //Start downloading the object asynchronously                    
1211
                    await client.DownloadFileTaskAsync(uri, fileName, cancellationToken,progress).ConfigureAwait(false);
1212

    
1213
                    //Once the download completes
1214
                    //Delete the local client object
1215
                }
1216
                //And report failure or completion
1217
            }
1218
            catch (Exception exc)
1219
            {
1220
                Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);
1221
                throw;
1222
            }
1223

    
1224
            Log.InfoFormat("[GET] END {0}", objectName);                                             
1225

    
1226

    
1227
        }
1228

    
1229
        public async Task<IList<string>> PutHashMap(string account, Uri container, Uri objectName, TreeHash hash)
1230
        {
1231
            if (container == null)
1232
                throw new ArgumentNullException("container", "The container property can't be empty");
1233
            if (container.IsAbsoluteUri)
1234
                throw new ArgumentException("The container must be relative","container");
1235
            if (objectName == null)
1236
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
1237
            if (objectName.IsAbsoluteUri)
1238
                throw new ArgumentException("The objectName must be relative","objectName");
1239
            if (hash == null)
1240
                throw new ArgumentNullException("hash");
1241
            if (String.IsNullOrWhiteSpace(Token))
1242
                throw new InvalidOperationException("Invalid Token");
1243
            if (StorageUrl == null)
1244
                throw new InvalidOperationException("Invalid Storage Url");
1245
            Contract.EndContractBlock();
1246

    
1247
            
1248

    
1249
            //The container and objectName are relative names. They are joined with the client's
1250
            //BaseAddress to create the object's absolute address
1251

    
1252
            var targetUri = GetTargetUri(account).Combine(container).Combine(objectName);
1253
  
1254

    
1255
            var uri = new Uri(String.Format("{0}?format=json&hashmap",targetUri),UriKind.Absolute);
1256

    
1257
            
1258
            //Send the tree hash as Json to the server            
1259
            var jsonHash = hash.ToJson();
1260
            if (Log.IsDebugEnabled)
1261
                Log.DebugFormat("Hashes:\r\n{0}", jsonHash);
1262

    
1263
            var mimeType = objectName.GetMimeType();
1264

    
1265
            var message = new HttpRequestMessage(HttpMethod.Put, uri)
1266
            {
1267
                Content = new StringContent(jsonHash)
1268
            };
1269
            message.Content.Headers.ContentType = mimeType;
1270
            message.Headers.Add("ETag",hash.TopHash.ToHashString());
1271
            
1272
            
1273
            //Don't use a timeout because putting the hashmap may be a long process
1274

    
1275
            using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3).ConfigureAwait(false))
1276
            {
1277
                var empty = (IList<string>)new List<string>();
1278
                
1279
                switch (response.StatusCode)
1280
                {
1281
                    case HttpStatusCode.Created:
1282
                        //The server will respond either with 201-created if all blocks were already on the server
1283
                        return empty;
1284
                    case HttpStatusCode.Conflict:
1285
                        //or with a 409-conflict and return the list of missing parts
1286
                        using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
1287
                        using(var reader=stream.GetLoggedReader(Log))
1288
                        {                            
1289
                            var serializer = new JsonSerializer();                            
1290
                            serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);
1291
                            var hashes = (List<string>)serializer.Deserialize(reader, typeof(List<string>));
1292
                            return hashes;
1293
                        }                        
1294
                    default:
1295
                        //All other cases are unexpected
1296
                        //Ensure that failure codes raise exceptions
1297
                        response.EnsureSuccessStatusCode();
1298
                        //And log any other codes as warngings, but continute processing
1299
                        Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",response.StatusCode,response.ReasonPhrase);
1300
                        return empty;
1301
                }
1302
            }
1303

    
1304
        }
1305

    
1306

    
1307
        public async Task<byte[]> GetBlock(string account, Uri container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)
1308
        {
1309
            if (String.IsNullOrWhiteSpace(Token))
1310
                throw new InvalidOperationException("Invalid Token");
1311
            if (StorageUrl == null)
1312
                throw new InvalidOperationException("Invalid Storage Url");
1313
            if (container == null)
1314
                throw new ArgumentNullException("container", "The container property can't be empty");
1315
            if (container.IsAbsoluteUri)
1316
                throw new ArgumentException("The container must be relative","container");
1317
            if (relativeUrl == null)
1318
                throw new ArgumentNullException("relativeUrl");
1319
            if (end.HasValue && end < 0)
1320
                throw new ArgumentOutOfRangeException("end");
1321
            if (start < 0)
1322
                throw new ArgumentOutOfRangeException("start");
1323
            Contract.EndContractBlock();
1324

    
1325

    
1326
            var targetUri = GetTargetUri(account).Combine(container).Combine(relativeUrl);
1327
            var message = new HttpRequestMessage(HttpMethod.Get, targetUri);
1328
            //Don't add a range if start=0, end=null (empty files)
1329
            if (start!=0 || end!=null)
1330
                message.Headers.Range=new RangeHeaderValue(start,end);
1331

    
1332
            //Don't use a timeout because putting the hashmap may be a long process
1333

    
1334
            IProgress<DownloadArgs> progress = new Progress<DownloadArgs>(args =>
1335
                {
1336
                    Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
1337
                                    targetUri.Segments.Last(), args.ProgressPercentage,
1338
                                    args.BytesReceived,
1339
                                    args.TotalBytesToReceive);
1340

    
1341
                    if (DownloadProgressChanged!=null)
1342
                        DownloadProgressChanged(this,  args);
1343
                });
1344

    
1345

    
1346
            using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, false,HttpCompletionOption.ResponseHeadersRead,
1347
                                                          cancellationToken).ConfigureAwait(false))
1348
            using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
1349
            {
1350
                long totalSize = response.Content.Headers.ContentLength ?? 0;
1351
                    byte[] buffer,streambuf;
1352
                    lock (_bufferManager)
1353
                    {
1354
                        buffer = _bufferManager.TakeBuffer(65536);
1355
                        streambuf = _bufferManager.TakeBuffer((int)totalSize);
1356
                    }
1357

    
1358
                using (var targetStream = new MemoryStream(streambuf))
1359
                {
1360

    
1361
                    long total = 0;
1362
                    try
1363
                    {
1364

    
1365
                        int read;
1366
                        while ((read = await stream.ReadAsync(buffer, 0, buffer.Length).ConfigureAwait(false)) != 0)
1367
                        {
1368
                            total += read;
1369
                            progress.Report(new DownloadArgs(total, totalSize));
1370
                            await targetStream.WriteAsync(buffer, 0, read).ConfigureAwait(false);
1371
                        }
1372
                    }
1373
                    finally
1374
                    {
1375
                        lock (_bufferManager)
1376
                        {
1377
                            _bufferManager.ReturnBuffer(buffer);
1378
                            _bufferManager.ReturnBuffer(streambuf);
1379
                        }
1380
                    }
1381
                    var result = targetStream.ToArray();
1382
                    return result;
1383
                }
1384
            }
1385

    
1386
        }
1387

    
1388
        public event EventHandler<UploadArgs> UploadProgressChanged;
1389
        public event EventHandler<DownloadArgs> DownloadProgressChanged;
1390
        
1391

    
1392
        public async Task PostBlock(string account, Uri container, byte[] block, int offset, int count,string blockHash,CancellationToken token)
1393
        {
1394
            if (container == null)
1395
                throw new ArgumentNullException("container", "The container property can't be empty");
1396
            if (container.IsAbsoluteUri)
1397
                throw new ArgumentException("The container must be relative","container");
1398
            if (block == null)
1399
                throw new ArgumentNullException("block");
1400
            if (offset < 0 || offset >= block.Length)
1401
                throw new ArgumentOutOfRangeException("offset");
1402
            if (count < 0 || count > block.Length)
1403
                throw new ArgumentOutOfRangeException("count");
1404
            if (String.IsNullOrWhiteSpace(Token))
1405
                throw new InvalidOperationException("Invalid Token");
1406
            if (StorageUrl == null)
1407
                throw new InvalidOperationException("Invalid Storage Url");                        
1408
            Contract.EndContractBlock();
1409

    
1410

    
1411
            try
1412
            {
1413
                var containerUri = GetTargetUri(account).Combine(container);
1414
                var targetUri = new Uri(String.Format("{0}?update", containerUri));
1415

    
1416

    
1417
                //Don't use a timeout because putting the hashmap may be a long process
1418

    
1419

    
1420
                Log.InfoFormat("[BLOCK POST] START");
1421

    
1422

    
1423
                var progress = new Progress<UploadArgs>(args =>
1424
                {
1425
                    Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
1426
                        args.ProgressPercentage,
1427
                        args.BytesSent,
1428
                        args.TotalBytesToSend);
1429
                    if (UploadProgressChanged != null)
1430
                        UploadProgressChanged(this,args);
1431
                });
1432

    
1433
                var message = new HttpRequestMessage(HttpMethod.Post, targetUri)
1434
                                  {
1435
                                      Content = new ByteArrayContentWithProgress(block, offset, count,progress)
1436
                                  };
1437
                message.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(@"application/octet-stream");
1438

    
1439
                //Send the block
1440
                using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3,false,HttpCompletionOption.ResponseContentRead,token).ConfigureAwait(false))
1441
                {                    
1442
                    Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
1443
                    response.EnsureSuccessStatusCode();
1444
                    var responseHash = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
1445
                    var cleanHash = responseHash.TrimEnd();
1446
                    Debug.Assert(blockHash==cleanHash);
1447
                    if (!blockHash.Equals(cleanHash, StringComparison.OrdinalIgnoreCase))
1448
                        Log.ErrorFormat("Block hash mismatch posting to [{0}]:[{1}], expected [{2}] but was [{3}]", account, container, blockHash, cleanHash);
1449
                }
1450
                Log.InfoFormat("[BLOCK POST] END");               
1451
            }
1452
            catch (TaskCanceledException )
1453
            {
1454
                Log.Info("Aborting block");
1455
                throw;
1456
            }
1457
            catch (Exception exc)
1458
            {
1459
                Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);
1460
                throw;
1461
            }
1462
        }
1463

    
1464
        public async Task PostBlock(string account, Uri container, string filePath, long offset, int count, string blockHash, CancellationToken token)
1465
        {
1466
            if (container == null)
1467
                throw new ArgumentNullException("container", "The container property can't be empty");
1468
            if (container.IsAbsoluteUri)
1469
                throw new ArgumentException("The container must be relative", "container");
1470
            if (String.IsNullOrWhiteSpace(filePath))
1471
                throw new ArgumentNullException("filePath");
1472
            if (!File.Exists(filePath))
1473
                throw new FileNotFoundException("Missing file","filePath");
1474
            if (String.IsNullOrWhiteSpace(Token))
1475
                throw new InvalidOperationException("Invalid Token");
1476
            if (StorageUrl == null)
1477
                throw new InvalidOperationException("Invalid Storage Url");
1478
            Contract.EndContractBlock();
1479

    
1480

    
1481
            try
1482
            {
1483
                var containerUri = GetTargetUri(account).Combine(container);
1484
                var targetUri = new Uri(String.Format("{0}?update", containerUri));
1485

    
1486

    
1487
                //Don't use a timeout because putting the hashmap may be a long process
1488

    
1489

    
1490
                Log.InfoFormat("[BLOCK POST] START");
1491

    
1492

    
1493
                var progress = new Progress<UploadArgs>(args =>
1494
                {
1495
                    Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2} at {3:###,} Kbps ",
1496
                        args.ProgressPercentage,
1497
                        args.BytesSent,
1498
                        args.TotalBytesToSend,args.Speed);
1499
                    if (UploadProgressChanged != null)
1500
                        UploadProgressChanged(this, args);
1501
                });
1502

    
1503
                var message = new HttpRequestMessage(HttpMethod.Post, targetUri)
1504
                {
1505
                    Content = new FileBlockContent(filePath, offset, count, progress)
1506
                };
1507
                message.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(@"application/octet-stream");
1508

    
1509
                //Send the block
1510
                using (var response = await _baseHttpClientNoTimeout.SendAsyncWithRetries(message, 3, false, HttpCompletionOption.ResponseContentRead, token).ConfigureAwait(false))
1511
                {
1512
                    Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
1513
                    response.EnsureSuccessStatusCode();
1514
                    var responseHash = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
1515
                    var cleanHash = responseHash.TrimEnd();
1516
                    Debug.Assert(blockHash == cleanHash);
1517
                    if (!blockHash.Equals(cleanHash, StringComparison.OrdinalIgnoreCase))
1518
                        Log.ErrorFormat("Block hash mismatch posting to [{0}]:[{1}], expected [{2}] but was [{3}]", account, container, blockHash, cleanHash);
1519
                }
1520
                Log.InfoFormat("[BLOCK POST] END");
1521
            }
1522
            catch (TaskCanceledException)
1523
            {
1524
                Log.Info("Aborting block");
1525
                throw;
1526
            }                
1527
            catch (Exception exc)
1528
            {
1529
                Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exc);
1530
                throw;
1531
            }
1532
        }
1533

    
1534

    
1535
        public async Task<TreeHash> GetHashMap(string account, Uri container, Uri objectName)
1536
        {
1537
            if (container == null)
1538
                throw new ArgumentNullException("container", "The container property can't be empty");
1539
            if (container.IsAbsoluteUri)
1540
                throw new ArgumentException("The container must be relative","container");
1541
            if (objectName == null)
1542
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
1543
            if (objectName.IsAbsoluteUri)
1544
                throw new ArgumentException("The objectName must be relative","objectName");
1545
            if (String.IsNullOrWhiteSpace(Token))
1546
                throw new InvalidOperationException("Invalid Token");
1547
            if (StorageUrl == null)
1548
                throw new InvalidOperationException("Invalid Storage Url");
1549
            Contract.EndContractBlock();
1550

    
1551
            try
1552
            {
1553

    
1554
                var objectUri = GetTargetUri(account).Combine(container).Combine(objectName);
1555
                var targetUri = new Uri(String.Format("{0}?format=json&hashmap", objectUri));
1556

    
1557
                //Start downloading the object asynchronously
1558
                var json = await GetStringAsync(targetUri, "").ConfigureAwait(false);
1559
                var treeHash = TreeHash.Parse(json);
1560
                Log.InfoFormat("[GET HASH] END {0}", objectName);
1561
                return treeHash;
1562

    
1563
            }
1564
            catch (Exception exc)
1565
            {
1566
                Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
1567
                throw;
1568
            }
1569

    
1570
        }
1571

    
1572

    
1573
        /// <summary>
1574
        /// 
1575
        /// </summary>
1576
        /// <param name="account"></param>
1577
        /// <param name="container"></param>
1578
        /// <param name="objectName"></param>
1579
        /// <param name="fileName"></param>
1580
        /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
1581
        /// <param name="contentType"> </param>
1582
        /// <remarks>>This method should have no timeout or a very long one</remarks>
1583
        public async Task PutObject(string account, Uri container, Uri objectName, string fileName, string hash = Signature.MERKLE_EMPTY, string contentType = "application/octet-stream")
1584
        {
1585
            if (container == null)
1586
                throw new ArgumentNullException("container", "The container property can't be empty");
1587
            if (container.IsAbsoluteUri)
1588
                throw new ArgumentException("The container must be relative","container");
1589
            if (objectName == null)
1590
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
1591
            if (objectName.IsAbsoluteUri)
1592
                throw new ArgumentException("The objectName must be relative","objectName");
1593
            if (String.IsNullOrWhiteSpace(fileName))
1594
                throw new ArgumentNullException("fileName", "The fileName property can't be empty");
1595
            try
1596
            {
1597

    
1598
                using (var client = new RestClient(_baseClient) { Timeout = 0 })
1599
                {
1600
                    if (!String.IsNullOrWhiteSpace(account))
1601
                        client.BaseAddress = GetAccountUrl(account);
1602

    
1603
                    var builder = client.GetAddressBuilder(container, objectName);
1604
                    var uri = builder.Uri;
1605

    
1606
                    string etag = hash ;
1607

    
1608
                    client.Headers.Add("Content-Type", contentType);
1609
                    if (contentType!=ObjectInfo.CONTENT_TYPE_DIRECTORY)
1610
                        client.Headers.Add("ETag", etag);
1611

    
1612

    
1613
                    Log.InfoFormat("[PUT] START {0}", objectName);
1614
                    client.UploadProgressChanged += (sender, args) =>
1615
                                                        {
1616
                                                            using (ThreadContext.Stacks["PUT"].Push("Progress"))
1617
                                                            {
1618
                                                                Log.InfoFormat("{0} {1}% {2} of {3}", fileName,
1619
                                                                               args.ProgressPercentage,
1620
                                                                               args.BytesSent, args.TotalBytesToSend);
1621
                                                            }
1622
                                                        };
1623

    
1624
                    client.UploadFileCompleted += (sender, args) =>
1625
                                                      {
1626
                                                          using (ThreadContext.Stacks["PUT"].Push("Progress"))
1627
                                                          {
1628
                                                              Log.InfoFormat("Completed {0}", fileName);
1629
                                                          }
1630
                                                      }; 
1631
                    
1632
                    if (contentType==ObjectInfo.CONTENT_TYPE_DIRECTORY)
1633
                        await client.UploadDataTaskAsync(uri, "PUT", new byte[0]).ConfigureAwait(false);
1634
                    else
1635
                        await client.UploadFileTaskAsync(uri, "PUT", fileName).ConfigureAwait(false);
1636
                }
1637

    
1638
                Log.InfoFormat("[PUT] END {0}", objectName);
1639
            }
1640
            catch (Exception exc)
1641
            {
1642
                Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
1643
                throw;
1644
            }                
1645

    
1646
        }
1647
        
1648
        public async Task MoveObject(string account, Uri sourceContainer, Uri oldObjectName, Uri targetContainer, Uri newObjectName)
1649
        {
1650
            if (sourceContainer == null)
1651
                throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");
1652
            if (sourceContainer.IsAbsoluteUri)
1653
                throw new ArgumentException("The sourceContainer must be relative","sourceContainer");
1654
            if (oldObjectName == null)
1655
                throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
1656
            if (oldObjectName.IsAbsoluteUri)
1657
                throw new ArgumentException("The oldObjectName must be relative","oldObjectName");
1658
            if (targetContainer == null)
1659
                throw new ArgumentNullException("targetContainer", "The targetContainer property can't be empty");
1660
            if (targetContainer.IsAbsoluteUri)
1661
                throw new ArgumentException("The targetContainer must be relative","targetContainer");
1662
            if (newObjectName == null)
1663
                throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
1664
            if (newObjectName.IsAbsoluteUri)
1665
                throw new ArgumentException("The newObjectName must be relative","newObjectName");
1666
            Contract.EndContractBlock();
1667

    
1668
            var baseUri = GetTargetUri(account);
1669
            var targetUri = baseUri.Combine(targetContainer).Combine(newObjectName);
1670
            var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, oldObjectName),UriKind.Relative);
1671

    
1672
            var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
1673
            message.Headers.Add("X-Move-From", sourceUri.ToString());
1674
            using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3).ConfigureAwait(false))
1675
            {
1676
                var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1677
                if (!expectedCodes.Contains(response.StatusCode))
1678
                    throw CreateWebException("MoveObject", response.StatusCode);
1679
            }
1680
        }
1681

    
1682
        public async Task DeleteObject(string account, Uri sourceContainer, Uri objectName, bool isDirectory)
1683
        {
1684
            if (sourceContainer == null)
1685
                throw new ArgumentNullException("sourceContainer", "The sourceContainer property can't be empty");
1686
            if (sourceContainer.IsAbsoluteUri)
1687
                throw new ArgumentException("The sourceContainer must be relative","sourceContainer");
1688
            if (objectName == null)
1689
                throw new ArgumentNullException("objectName", "The objectName property can't be empty");
1690
            if (objectName.IsAbsoluteUri)
1691
                throw new ArgumentException("The objectName must be relative","objectName");
1692
            Contract.EndContractBlock();
1693

    
1694

    
1695

    
1696
            var sourceUri = new Uri(String.Format("/{0}/{1}", sourceContainer, objectName),UriKind.Relative);
1697

    
1698
            
1699
            if (objectName.OriginalString.EndsWith(".ignore"))
1700
                using(var response = await _baseHttpClient.DeleteAsync(sourceUri)){}
1701
            else
1702
            {
1703
                var relativeUri = new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName),
1704
                                                UriKind.Relative);
1705

    
1706
/*
1707
                var relativeUri = isDirectory
1708
                                      ? new Uri(
1709
                                            String.Format("{0}/{1}?delimiter=/", FolderConstants.TrashContainer,
1710
                                                          objectName), UriKind.Relative)
1711
                                      : new Uri(String.Format("{0}/{1}", FolderConstants.TrashContainer, objectName),
1712
                                                UriKind.Relative);
1713

    
1714
*/
1715
                var targetUri = GetTargetUri(account).Combine(relativeUri);
1716

    
1717

    
1718
                var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
1719
                message.Headers.Add("X-Move-From", sourceUri.ToString());
1720

    
1721
                Log.InfoFormat("[TRASH] [{0}] to [{1}]", sourceUri, targetUri);
1722
                using (var response = await _baseHttpClient.SendAsyncWithRetries(message, 3))
1723
                {
1724
                    var expectedCodes = new[]
1725
                                            {
1726
                                                HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,
1727
                                                HttpStatusCode.NotFound
1728
                                            };
1729
                    if (!expectedCodes.Contains(response.StatusCode))
1730
                        throw CreateWebException("DeleteObject", response.StatusCode);
1731
                }
1732
            }
1733
/*
1734
            
1735

    
1736
            var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
1737
/*
1738
            if (isDirectory)
1739
                targetUrl = targetUrl + "?delimiter=/";
1740
#1#
1741

    
1742
            var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
1743

    
1744
            using (var client = new RestClient(_baseClient))
1745
            {
1746
                if (!String.IsNullOrWhiteSpace(account))
1747
                    client.BaseAddress = GetAccountUrl(account);
1748

    
1749
                client.Headers.Add("X-Move-From", sourceUrl);
1750
                client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
1751
                Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);
1752
                client.PutWithRetry(new Uri(targetUrl,UriKind.Relative), 3);
1753

    
1754
                var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
1755
                if (!expectedCodes.Contains(client.StatusCode))
1756
                    throw CreateWebException("DeleteObject", client.StatusCode);
1757
            }
1758
*/
1759
        }
1760

    
1761
      
1762
        private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
1763
        {
1764
            return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
1765
        }
1766

    
1767

    
1768
        public async Task<bool> CanUpload(string account, ObjectInfo cloudFile)
1769
        {
1770
            Contract.Requires(!String.IsNullOrWhiteSpace(account));
1771
            Contract.Requires(cloudFile!=null);
1772

    
1773
                var parts = cloudFile.Name.ToString().Split('/');
1774
                var folder = String.Join("/", parts,0,parts.Length-1);
1775

    
1776
                var fileName = String.Format("{0}/{1}.pithos.ignore", folder, Guid.NewGuid());
1777
                var fileUri=fileName.ToEscapedUri();                                            
1778

    
1779
                try
1780
                {
1781
                    var relativeUri = cloudFile.Container.Combine(fileUri);
1782
                    var targetUri = GetTargetUri(account).Combine(relativeUri);
1783
                    var message = new HttpRequestMessage(HttpMethod.Put, targetUri);
1784
                    message.Content.Headers.ContentType =new MediaTypeHeaderValue("application/octet-stream");
1785
                    var response=await _baseHttpClient.SendAsyncWithRetries(message, 3);                    
1786
                    var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1787
                    var result=(expectedCodes.Contains(response.StatusCode));
1788
                    await DeleteObject(account, cloudFile.Container, fileUri, cloudFile.IsDirectory);
1789
                    return result;
1790
                }
1791
                catch
1792
                {
1793
                    return false;
1794
                }
1795
            
1796
        }
1797

    
1798
        ~CloudFilesClient()
1799
        {
1800
            Dispose(false);
1801
        }
1802

    
1803
        public void Dispose()
1804
        {
1805
            Dispose(true);
1806
            GC.SuppressFinalize(this);
1807
        }
1808

    
1809
        protected virtual void Dispose(bool disposing)
1810
        {
1811
            if (disposing)
1812
            {
1813
                if (_httpClientHandler!=null)
1814
                    _httpClientHandler.Dispose();
1815
                if (_baseClient!=null)
1816
                    _baseClient.Dispose();
1817
                if(_baseHttpClient!=null)
1818
                    _baseHttpClient.Dispose();
1819
                if (_baseHttpClientNoTimeout!=null)
1820
                    _baseHttpClientNoTimeout.Dispose();
1821
            }
1822
            _httpClientHandler = null;
1823
            _baseClient = null;
1824
            _baseHttpClient = null;
1825
            _baseHttpClientNoTimeout = null;
1826
        }
1827

    
1828
        public async Task<string> ResolveName(Guid accountToken)
1829
        {
1830
            string format = string.Format("{{\"uuids\":[\"{0}\"]}}", accountToken);
1831
            var content = new StringContent(format,Encoding.UTF8);
1832
            //content.Headers.ContentType=new MediaTypeHeaderValue("text/html; charset=utf-8");
1833
            string catalogEntry;
1834
            //var catalogUrl = new Uri(_baseHttpClient.BaseAddress.Scheme + "://" +_baseHttpClient.BaseAddress.Host,UriKind.Absolute).Combine("user_catalogs");
1835
            var catalogUrl = new Uri("https://accounts.okeanos.grnet.gr/account/v1.0/user_catalogs");
1836
            using (var response = await _baseHttpClient.PostAsync(catalogUrl, content).ConfigureAwait(false))
1837
            {
1838
                catalogEntry=await response.Content.ReadAsStringAsync().ConfigureAwait(false);
1839
            }
1840

    
1841
            var entry = (JContainer)JsonConvert.DeserializeObject(catalogEntry);
1842
            string key = accountToken.ToString();
1843
            return (string)entry["uuid_catalog"][key];
1844

    
1845
        }
1846

    
1847
        public async Task<Guid> ResolveToken(string displayName)
1848
        {
1849
            string format = string.Format("{{\"displaynames\":[\"{0}\"]}}", displayName);
1850
            var content = new StringContent(format,Encoding.UTF8);
1851
            //content.Headers.ContentType=new MediaTypeHeaderValue("text/html; charset=utf-8");
1852
            string catalogEntry;
1853
            //var catalogUrl = new Uri(_baseHttpClient.BaseAddress.Scheme + "://" +_baseHttpClient.BaseAddress.Host,UriKind.Absolute).Combine("user_catalogs");
1854
            var catalogUrl = new Uri("https://accounts.okeanos.grnet.gr/account/v1.0/user_catalogs");
1855
            using (var response = await _baseHttpClient.PostAsync(catalogUrl, content).ConfigureAwait(false))
1856
            {
1857
                catalogEntry=await response.Content.ReadAsStringAsync().ConfigureAwait(false);
1858
            }
1859

    
1860
            var entry = (JContainer)JsonConvert.DeserializeObject(catalogEntry);
1861
            return new Guid((string)entry["displayname_catalog"][displayName]);
1862

    
1863
        }
1864
    }
1865

    
1866
    public class ShareAccountInfo
1867
    {
1868
        public DateTime? last_modified { get; set; }
1869
        public string name { get; set; }
1870
    }
1871
}