Now closing web request immediatelly after executing a request that has no content.
[pithos-ms-client] / trunk / Pithos.Network / CloudFilesClient.cs
1 // **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
2 //
3 // The class provides methods to upload/download files, delete files, manage containers
4
5
6 using System;
7 using System.Collections.Generic;
8 using System.Collections.Specialized;
9 using System.ComponentModel.Composition;
10 using System.Diagnostics;
11 using System.Diagnostics.Contracts;
12 using System.IO;
13 using System.Linq;
14 using System.Net;
15 using System.Security.Cryptography;
16 using System.Text;
17 using System.Threading.Tasks;
18 using Newtonsoft.Json;
19 using Pithos.Interfaces;
20 using log4net;
21
22 namespace Pithos.Network
23 {
24     [Export(typeof(ICloudClient))]
25     public class CloudFilesClient:ICloudClient
26     {
27         //CloudFilesClient uses *_baseClient* internally to communicate with the server
28         //RestClient provides a REST-friendly interface over the standard WebClient.
29         private RestClient _baseClient;
30         
31
32         //During authentication the client provides a UserName 
33         public string UserName { get; set; }
34         
35         //and and ApiKey to the server
36         public string ApiKey { get; set; }
37         
38         //And receives an authentication Token. This token must be provided in ALL other operations,
39         //in the X-Auth-Token header
40         private string _token;
41         public string Token
42         {
43             get { return _token; }
44             set
45             {
46                 _token = value;
47                 _baseClient.Headers["X-Auth-Token"] = value;
48             }
49         }
50
51         //The client also receives a StorageUrl after authentication. All subsequent operations must
52         //use this url
53         public Uri StorageUrl { get; set; }
54
55
56         protected Uri RootAddressUri { get; set; }
57
58         private Uri _proxy;
59         public Uri Proxy
60         {
61             get { return _proxy; }
62             set
63             {
64                 _proxy = value;
65                 if (_baseClient != null)
66                     _baseClient.Proxy = new WebProxy(value);                
67             }
68         }
69
70         public double DownloadPercentLimit { get; set; }
71         public double UploadPercentLimit { get; set; }
72
73         public string AuthenticationUrl { get; set; }
74
75  
76         public string VersionPath
77         {
78             get { return UsePithos ? "v1" : "v1.0"; }
79         }
80
81         public bool UsePithos { get; set; }
82
83
84         private static readonly ILog Log = LogManager.GetLogger("CloudFilesClient");
85
86         public CloudFilesClient(string userName, string apiKey)
87         {
88             UserName = userName;
89             ApiKey = apiKey;
90         }
91
92         public CloudFilesClient(AccountInfo accountInfo)
93         {
94             if (accountInfo==null)
95                 throw new ArgumentNullException("accountInfo");
96             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
97             Contract.Ensures(StorageUrl != null);
98             Contract.Ensures(_baseClient != null);
99             Contract.Ensures(RootAddressUri != null);
100             Contract.EndContractBlock();
101
102             _baseClient = new RestClient
103             {
104                 BaseAddress = accountInfo.StorageUri.ToString(),
105                 Timeout = 10000,
106                 Retries = 3
107             };
108             StorageUrl = accountInfo.StorageUri;
109             Token = accountInfo.Token;
110             UserName = accountInfo.UserName;
111
112             //Get the root address (StorageUrl without the account)
113             var storageUrl = StorageUrl.AbsoluteUri;
114             var usernameIndex = storageUrl.LastIndexOf(UserName);
115             var rootUrl = storageUrl.Substring(0, usernameIndex);
116             RootAddressUri = new Uri(rootUrl);
117         }
118
119
120         public AccountInfo Authenticate()
121         {
122             if (String.IsNullOrWhiteSpace(UserName))
123                 throw new InvalidOperationException("UserName is empty");
124             if (String.IsNullOrWhiteSpace(ApiKey))
125                 throw new InvalidOperationException("ApiKey is empty");
126             if (String.IsNullOrWhiteSpace(AuthenticationUrl))
127                 throw new InvalidOperationException("AuthenticationUrl is empty");
128             Contract.Ensures(!String.IsNullOrWhiteSpace(Token));
129             Contract.Ensures(StorageUrl != null);
130             Contract.Ensures(_baseClient != null);
131             Contract.Ensures(RootAddressUri != null);
132             Contract.EndContractBlock();
133
134
135             Log.InfoFormat("[AUTHENTICATE] Start for {0}", UserName);
136
137             var groups = new List<Group>();
138
139             using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
140             {
141                 if (Proxy != null)
142                     authClient.Proxy = new WebProxy(Proxy);
143
144                 Contract.Assume(authClient.Headers!=null);
145
146                 authClient.Headers.Add("X-Auth-User", UserName);
147                 authClient.Headers.Add("X-Auth-Key", ApiKey);
148
149                 authClient.DownloadStringWithRetry(VersionPath, 3);
150
151                 authClient.AssertStatusOK("Authentication failed");
152
153                 var storageUrl = authClient.GetHeaderValue("X-Storage-Url");
154                 if (String.IsNullOrWhiteSpace(storageUrl))
155                     throw new InvalidOperationException("Failed to obtain storage url");
156                 
157                 _baseClient = new RestClient
158                 {
159                     BaseAddress = storageUrl,
160                     Timeout = 10000,
161                     Retries = 3
162                 };
163
164                 StorageUrl = new Uri(storageUrl);
165                 
166                 //Get the root address (StorageUrl without the account)
167                 var usernameIndex=storageUrl.LastIndexOf(UserName);
168                 var rootUrl = storageUrl.Substring(0, usernameIndex);
169                 RootAddressUri = new Uri(rootUrl);
170                 
171                 var token = authClient.GetHeaderValue("X-Auth-Token");
172                 if (String.IsNullOrWhiteSpace(token))
173                     throw new InvalidOperationException("Failed to obtain token url");
174                 Token = token;
175
176                /* var keys = authClient.ResponseHeaders.AllKeys.AsQueryable();
177                 groups = (from key in keys
178                             where key.StartsWith("X-Account-Group-")
179                             let name = key.Substring(16)
180                             select new Group(name, authClient.ResponseHeaders[key]))
181                         .ToList();
182                     
183 */
184             }
185
186             Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
187             
188
189             return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};            
190
191         }
192
193
194
195         public IList<ContainerInfo> ListContainers(string account)
196         {
197             using (var client = new RestClient(_baseClient))
198             {
199                 if (!String.IsNullOrWhiteSpace(account))
200                     client.BaseAddress = GetAccountUrl(account);
201                 
202                 client.Parameters.Clear();
203                 client.Parameters.Add("format", "json");
204                 var content = client.DownloadStringWithRetry("", 3);
205                 client.AssertStatusOK("List Containers failed");
206
207                 if (client.StatusCode == HttpStatusCode.NoContent)
208                     return new List<ContainerInfo>();
209                 var infos = JsonConvert.DeserializeObject<IList<ContainerInfo>>(content);
210                 
211                 foreach (var info in infos)
212                 {
213                     info.Account = account;
214                 }
215                 return infos;
216             }
217
218         }
219
220         private string GetAccountUrl(string account)
221         {
222             return new Uri(this.RootAddressUri, new Uri(account,UriKind.Relative)).AbsoluteUri;
223         }
224
225         public IList<ShareAccountInfo> ListSharingAccounts(DateTime? since=null)
226         {
227             using (log4net.ThreadContext.Stacks["Share"].Push("List Accounts"))
228             {
229                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
230
231                 using (var client = new RestClient(_baseClient))
232                 {
233                     client.Parameters.Clear();
234                     client.Parameters.Add("format", "json");
235                     client.IfModifiedSince = since;
236
237                     //Extract the username from the base address
238                     client.BaseAddress = RootAddressUri.AbsoluteUri;
239
240                     var content = client.DownloadStringWithRetry(@"", 3);
241
242                     client.AssertStatusOK("ListSharingAccounts failed");
243
244                     //If the result is empty, return an empty list,
245                     var infos = String.IsNullOrWhiteSpace(content)
246                                     ? new List<ShareAccountInfo>()
247                                 //Otherwise deserialize the account list into a list of ShareAccountInfos
248                                     : JsonConvert.DeserializeObject<IList<ShareAccountInfo>>(content);
249
250                     Log.DebugFormat("END");
251                     return infos;
252                 }
253             }
254         }
255
256         //Request listing of all objects in a container modified since a specific time.
257         //If the *since* value is missing, return all objects
258         public IList<ObjectInfo> ListSharedObjects(DateTime? since = null)
259         {
260
261             using (log4net.ThreadContext.Stacks["Share"].Push("List Objects"))
262             {
263                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
264
265                 var objects = new List<ObjectInfo>();
266                 var accounts = ListSharingAccounts(since);
267                 foreach (var account in accounts)
268                 {
269                     var containers = ListContainers(account.name);
270                     foreach (var container in containers)
271                     {
272                         var containerObjects = ListObjects(account.name, container.Name, null);
273                         objects.AddRange(containerObjects);
274                     }
275                 }
276                 if (Log.IsDebugEnabled) Log.DebugFormat("END");
277                 return objects;
278             }
279         }
280
281         public void SetTags(ObjectInfo target,IDictionary<string,string> tags)
282         {
283             if (String.IsNullOrWhiteSpace(Token))
284                 throw new InvalidOperationException("The Token is not set");
285             if (StorageUrl == null)
286                 throw new InvalidOperationException("The StorageUrl is not set");
287             if (target == null)
288                 throw new ArgumentNullException("target");
289             Contract.EndContractBlock();
290
291             using (log4net.ThreadContext.Stacks["Share"].Push("Share Object"))
292             {
293                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
294
295                 using (var client = new RestClient(_baseClient))
296                 {
297
298                     client.BaseAddress = GetAccountUrl(target.Account);
299
300                     client.Parameters.Clear();
301                     client.Parameters.Add("update", "");
302
303                     foreach (var tag in tags)
304                     {
305                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
306                         client.Headers.Add(headerTag, tag.Value);
307                     }
308                     
309                     var content = client.DownloadStringWithRetry(target.Container, 3);
310
311                     
312                     client.AssertStatusOK("SetTags failed");
313                     //If the status is NOT ACCEPTED we have a problem
314                     if (client.StatusCode != HttpStatusCode.Accepted)
315                     {
316                         Log.Error("Failed to set tags");
317                         throw new Exception("Failed to set tags");
318                     }
319
320                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
321                 }
322             }
323
324
325         }
326
327         public void ShareObject(string account, string container, string objectName, string shareTo, bool read, bool write)
328         {
329             if (String.IsNullOrWhiteSpace(Token))
330                 throw new InvalidOperationException("The Token is not set");
331             if (StorageUrl==null)
332                 throw new InvalidOperationException("The StorageUrl is not set");
333             if (String.IsNullOrWhiteSpace(container))
334                 throw new ArgumentNullException("container");
335             if (String.IsNullOrWhiteSpace(objectName))
336                 throw new ArgumentNullException("objectName");
337             if (String.IsNullOrWhiteSpace(account))
338                 throw new ArgumentNullException("account");
339             if (String.IsNullOrWhiteSpace(shareTo))
340                 throw new ArgumentNullException("shareTo");
341             Contract.EndContractBlock();
342
343             using (log4net.ThreadContext.Stacks["Share"].Push("Share Object"))
344             {
345                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
346                 
347                 using (var client = new RestClient(_baseClient))
348                 {
349
350                     client.BaseAddress = GetAccountUrl(account);
351
352                     client.Parameters.Clear();
353                     client.Parameters.Add("format", "json");
354
355                     string permission = "";
356                     if (write)
357                         permission = String.Format("write={0}", shareTo);
358                     else if (read)
359                         permission = String.Format("read={0}", shareTo);
360                     client.Headers.Add("X-Object-Sharing", permission);
361
362                     var content = client.DownloadStringWithRetry(container, 3);
363
364                     client.AssertStatusOK("ShareObject failed");
365
366                     //If the result is empty, return an empty list,
367                     var infos = String.IsNullOrWhiteSpace(content)
368                                     ? new List<ObjectInfo>()
369                                 //Otherwise deserialize the object list into a list of ObjectInfos
370                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
371
372                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
373                 }
374             }
375
376
377         }
378
379         public AccountInfo GetAccountPolicies(AccountInfo accountInfo)
380         {
381             if (accountInfo==null)
382                 throw new ArgumentNullException("accountInfo");
383             Contract.EndContractBlock();
384
385             using (log4net.ThreadContext.Stacks["Account"].Push("GetPolicies"))
386             {
387                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
388
389                 using (var client = new RestClient(_baseClient))
390                 {
391                     if (!String.IsNullOrWhiteSpace(accountInfo.UserName))
392                         client.BaseAddress = GetAccountUrl(accountInfo.UserName);
393
394                     client.Parameters.Clear();
395                     client.Parameters.Add("format", "json");                    
396                     client.Head(String.Empty, 3);
397
398                     var quotaValue=client.ResponseHeaders["X-Account-Policy-Quota"];
399                     var bytesValue= client.ResponseHeaders["X-Account-Bytes-Used"];
400
401                     long quota, bytes;
402                     if (long.TryParse(quotaValue, out quota))
403                         accountInfo.Quota = quota;
404                     if (long.TryParse(bytesValue, out bytes))
405                         accountInfo.BytesUsed = bytes;
406                     
407                     return accountInfo;
408
409                 }
410
411             }
412         }
413
414         public void UpdateMetadata(ObjectInfo objectInfo)
415         {
416             if (objectInfo == null)
417                 throw new ArgumentNullException("objectInfo");
418             Contract.EndContractBlock();
419
420             using (log4net.ThreadContext.Stacks["Objects"].Push("UpdateMetadata"))
421             {
422                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
423
424
425                 using(var client=new RestClient(_baseClient))
426                 {
427
428                     client.BaseAddress = GetAccountUrl(objectInfo.Account);
429                     
430                     client.Parameters.Clear();
431                     
432
433                     //Set Tags
434                     foreach (var tag in objectInfo.Tags)
435                     {
436                         var headerTag = String.Format("X-Object-Meta-{0}", tag.Key);
437                         client.Headers.Add(headerTag, tag.Value);
438                     }
439
440                     //Set Permissions
441
442                     var permissions=objectInfo.GetPermissionString();
443                     client.SetNonEmptyHeaderValue("X-Object-Sharing",permissions);
444
445                     client.SetNonEmptyHeaderValue("Content-Disposition",objectInfo.ContendDisposition);
446                     client.SetNonEmptyHeaderValue("Content-Encoding",objectInfo.ContentEncoding);
447                     client.SetNonEmptyHeaderValue("X-Object-Manifest",objectInfo.Manifest);
448                     var isPublic = objectInfo.IsPublic.ToString().ToLower();
449                     client.Headers.Add("X-Object-Public", isPublic);
450
451
452                     var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);
453                     var uri = uriBuilder.Uri;
454
455                     var content = client.UploadValues(uri,new NameValueCollection());
456
457
458                     client.AssertStatusOK("UpdateMetadata failed");
459                     //If the status is NOT ACCEPTED or OK we have a problem
460                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
461                     {
462                         Log.Error("Failed to update metadata");
463                         throw new Exception("Failed to update metadata");
464                     }
465
466                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
467                 }
468             }
469
470         }
471
472         public void UpdateMetadata(ContainerInfo containerInfo)
473         {
474             if (containerInfo == null)
475                 throw new ArgumentNullException("containerInfo");
476             Contract.EndContractBlock();
477
478             using (log4net.ThreadContext.Stacks["Containers"].Push("UpdateMetadata"))
479             {
480                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
481
482
483                 using(var client=new RestClient(_baseClient))
484                 {
485
486                     client.BaseAddress = GetAccountUrl(containerInfo.Account);
487                     
488                     client.Parameters.Clear();
489                     
490
491                     //Set Tags
492                     foreach (var tag in containerInfo.Tags)
493                     {
494                         var headerTag = String.Format("X-Container-Meta-{0}", tag.Key);
495                         client.Headers.Add(headerTag, tag.Value);
496                     }
497
498                     
499                     //Set Policies
500                     foreach (var policy in containerInfo.Policies)
501                     {
502                         var headerPolicy = String.Format("X-Container-Policy-{0}", policy.Key);
503                         client.Headers.Add(headerPolicy, policy.Value);
504                     }
505
506
507                     var uriBuilder = client.GetAddressBuilder(containerInfo.Name,"");
508                     var uri = uriBuilder.Uri;
509
510                     var content = client.UploadValues(uri,new NameValueCollection());
511
512
513                     client.AssertStatusOK("UpdateMetadata failed");
514                     //If the status is NOT ACCEPTED or OK we have a problem
515                     if (!(client.StatusCode == HttpStatusCode.Accepted || client.StatusCode == HttpStatusCode.OK))
516                     {
517                         Log.Error("Failed to update metadata");
518                         throw new Exception("Failed to update metadata");
519                     }
520
521                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
522                 }
523             }
524
525         }
526
527
528         public IList<ObjectInfo> ListObjects(string account, string container, DateTime? since = null)
529         {
530             if (String.IsNullOrWhiteSpace(container))
531                 throw new ArgumentNullException("container");
532             Contract.EndContractBlock();
533
534             using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
535             {
536                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
537
538                 using (var client = new RestClient(_baseClient))
539                 {
540                     if (!String.IsNullOrWhiteSpace(account))
541                         client.BaseAddress = GetAccountUrl(account);
542
543                     client.Parameters.Clear();
544                     client.Parameters.Add("format", "json");
545                     client.IfModifiedSince = since;
546                     var content = client.DownloadStringWithRetry(container, 3);
547
548                     client.AssertStatusOK("ListObjects failed");
549
550                     //If the result is empty, return an empty list,
551                     var infos = String.IsNullOrWhiteSpace(content)
552                                     ? new List<ObjectInfo>()
553                                 //Otherwise deserialize the object list into a list of ObjectInfos
554                                     : JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
555
556                     foreach (var info in infos)
557                     {
558                         info.Container = container;
559                         info.Account = account;
560                     }
561                     if (Log.IsDebugEnabled) Log.DebugFormat("START");
562                     return infos;
563                 }
564             }
565         }
566
567
568
569         public IList<ObjectInfo> ListObjects(string account, string container, string folder, DateTime? since = null)
570         {
571             if (String.IsNullOrWhiteSpace(container))
572                 throw new ArgumentNullException("container");
573             if (String.IsNullOrWhiteSpace(folder))
574                 throw new ArgumentNullException("folder");
575             Contract.EndContractBlock();
576
577             using (log4net.ThreadContext.Stacks["Objects"].Push("List"))
578             {
579                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
580
581                 using (var client = new RestClient(_baseClient))
582                 {
583                     if (!String.IsNullOrWhiteSpace(account))
584                         client.BaseAddress = GetAccountUrl(account);
585
586                     client.Parameters.Clear();
587                     client.Parameters.Add("format", "json");
588                     client.Parameters.Add("path", folder);
589                     client.IfModifiedSince = since;
590                     var content = client.DownloadStringWithRetry(container, 3);
591                     client.AssertStatusOK("ListObjects failed");
592
593                     var infos = JsonConvert.DeserializeObject<IList<ObjectInfo>>(content);
594
595                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
596                     return infos;
597                 }
598             }
599         }
600
601  
602         public bool ContainerExists(string account, string container)
603         {
604             if (String.IsNullOrWhiteSpace(container))
605                 throw new ArgumentNullException("container", "The container property can't be empty");
606             Contract.EndContractBlock();
607
608             using (log4net.ThreadContext.Stacks["Containters"].Push("Exists"))
609             {
610                 if (Log.IsDebugEnabled) Log.DebugFormat("START");
611
612                 using (var client = new RestClient(_baseClient))
613                 {
614                     if (!String.IsNullOrWhiteSpace(account))
615                         client.BaseAddress = GetAccountUrl(account);
616
617                     client.Parameters.Clear();
618                     client.Head(container, 3);
619                                         
620                     bool result;
621                     switch (client.StatusCode)
622                     {
623                         case HttpStatusCode.OK:
624                         case HttpStatusCode.NoContent:
625                             result=true;
626                             break;
627                         case HttpStatusCode.NotFound:
628                             result=false;
629                             break;
630                         default:
631                             throw CreateWebException("ContainerExists", client.StatusCode);
632                     }
633                     if (Log.IsDebugEnabled) Log.DebugFormat("END");
634
635                     return result;
636                 }
637                 
638             }
639         }
640
641         public bool ObjectExists(string account, string container, string objectName)
642         {
643             if (String.IsNullOrWhiteSpace(container))
644                 throw new ArgumentNullException("container", "The container property can't be empty");
645             if (String.IsNullOrWhiteSpace(objectName))
646                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
647             Contract.EndContractBlock();
648
649             using (var client = new RestClient(_baseClient))
650             {
651                 if (!String.IsNullOrWhiteSpace(account))
652                     client.BaseAddress = GetAccountUrl(account);
653
654                 client.Parameters.Clear();
655                 client.Head(container + "/" + objectName, 3);
656
657                 switch (client.StatusCode)
658                 {
659                     case HttpStatusCode.OK:
660                     case HttpStatusCode.NoContent:
661                         return true;
662                     case HttpStatusCode.NotFound:
663                         return false;
664                     default:
665                         throw CreateWebException("ObjectExists", client.StatusCode);
666                 }
667             }
668
669         }
670
671         public ObjectInfo GetObjectInfo(string account, string container, string objectName)
672         {
673             if (String.IsNullOrWhiteSpace(container))
674                 throw new ArgumentNullException("container", "The container property can't be empty");
675             if (String.IsNullOrWhiteSpace(objectName))
676                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
677             Contract.EndContractBlock();
678
679             using (log4net.ThreadContext.Stacks["Objects"].Push("GetObjectInfo"))
680             {                
681
682                 using (var client = new RestClient(_baseClient))
683                 {
684                     if (!String.IsNullOrWhiteSpace(account))
685                         client.BaseAddress = GetAccountUrl(account);
686                     try
687                     {
688                         client.Parameters.Clear();
689
690                         client.Head(container + "/" + objectName, 3);
691
692                         if (client.TimedOut)
693                             return ObjectInfo.Empty;
694
695                         switch (client.StatusCode)
696                         {
697                             case HttpStatusCode.OK:
698                             case HttpStatusCode.NoContent:
699                                 var keys = client.ResponseHeaders.AllKeys.AsQueryable();
700                                 var tags = client.GetMeta("X-Object-Meta-");
701                                 var extensions = (from key in keys
702                                                   where key.StartsWith("X-Object-") && !key.StartsWith("X-Object-Meta-")
703                                                   select new {Name = key, Value = client.ResponseHeaders[key]})
704                                     .ToDictionary(t => t.Name, t => t.Value);
705
706                                 var permissions=client.GetHeaderValue("X-Object-Sharing", true);
707                                 
708                                 
709                                 var info = new ObjectInfo
710                                                {
711                                                    Account = account,
712                                                    Container = container,
713                                                    Name = objectName,
714                                                    Hash = client.GetHeaderValue("ETag"),
715                                                    Content_Type = client.GetHeaderValue("Content-Type"),
716                                                    Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),
717                                                    Tags = tags,
718                                                    Last_Modified = client.LastModified,
719                                                    Extensions = extensions,
720                                                    ContentEncoding=client.GetHeaderValue("Content-Encoding",true),
721                                                    ContendDisposition = client.GetHeaderValue("Content-Disposition",true),
722                                                    Manifest=client.GetHeaderValue("X-Object-Manifest",true),
723                                                    PublicUrl=client.GetHeaderValue("X-Object-Public",true),                                                   
724                                                };
725                                 info.SetPermissions(permissions);
726                                 return info;
727                             case HttpStatusCode.NotFound:
728                                 return ObjectInfo.Empty;
729                             default:
730                                 throw new WebException(
731                                     String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
732                                                   objectName, client.StatusCode));
733                         }
734
735                     }
736                     catch (RetryException)
737                     {
738                         Log.WarnFormat("[RETRY FAIL] GetObjectInfo for {0} failed.");
739                         return ObjectInfo.Empty;
740                     }
741                     catch (WebException e)
742                     {
743                         Log.Error(
744                             String.Format("[FAIL] GetObjectInfo for {0} failed with unexpected status code {1}",
745                                           objectName, client.StatusCode), e);
746                         throw;
747                     }
748                 }                
749             }
750
751         }
752
753         public void CreateFolder(string account, string container, string folder)
754         {
755             if (String.IsNullOrWhiteSpace(container))
756                 throw new ArgumentNullException("container", "The container property can't be empty");
757             if (String.IsNullOrWhiteSpace(folder))
758                 throw new ArgumentNullException("folder", "The folder property can't be empty");
759             Contract.EndContractBlock();
760
761             var folderUrl=String.Format("{0}/{1}",container,folder);
762             using (var client = new RestClient(_baseClient))
763             {
764                 if (!String.IsNullOrWhiteSpace(account))
765                     client.BaseAddress = GetAccountUrl(account);
766
767                 client.Parameters.Clear();
768                 client.Headers.Add("Content-Type", @"application/directory");
769                 client.Headers.Add("Content-Length", "0");
770                 client.PutWithRetry(folderUrl, 3);
771
772                 if (client.StatusCode != HttpStatusCode.Created && client.StatusCode != HttpStatusCode.Accepted)
773                     throw CreateWebException("CreateFolder", client.StatusCode);
774             }
775         }
776
777      
778
779         public ContainerInfo GetContainerInfo(string account, string container)
780         {
781             if (String.IsNullOrWhiteSpace(container))
782                 throw new ArgumentNullException("container", "The container property can't be empty");
783             Contract.EndContractBlock();
784
785             using (var client = new RestClient(_baseClient))
786             {
787                 if (!String.IsNullOrWhiteSpace(account))
788                     client.BaseAddress = GetAccountUrl(account);                
789
790                 client.Head(container);
791                 switch (client.StatusCode)
792                 {
793                     case HttpStatusCode.OK:
794                     case HttpStatusCode.NoContent:
795                         var tags = client.GetMeta("X-Container-Meta-");
796                         var policies = client.GetMeta("X-Container-Policy-");
797
798                         var containerInfo = new ContainerInfo
799                                                 {
800                                                     Account=account,
801                                                     Name = container,
802                                                     Count =
803                                                         long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
804                                                     Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
805                                                     BlockHash = client.GetHeaderValue("X-Container-Block-Hash"),
806                                                     BlockSize=int.Parse(client.GetHeaderValue("X-Container-Block-Size")),
807                                                     Last_Modified=client.LastModified,
808                                                     Tags=tags,
809                                                     Policies=policies
810                                                 };
811                         
812
813                         return containerInfo;
814                     case HttpStatusCode.NotFound:
815                         return ContainerInfo.Empty;
816                     default:
817                         throw CreateWebException("GetContainerInfo", client.StatusCode);
818                 }
819             }
820         }
821
822         public void CreateContainer(string account, string container)
823         {
824             if (String.IsNullOrWhiteSpace(account))
825                 throw new ArgumentNullException("account");
826             if (String.IsNullOrWhiteSpace(container))
827                 throw new ArgumentNullException("container");
828             Contract.EndContractBlock();
829
830             using (var client = new RestClient(_baseClient))
831             {
832                 if (!String.IsNullOrWhiteSpace(account))
833                     client.BaseAddress = GetAccountUrl(account);
834
835                 client.PutWithRetry(container, 3);
836                 var expectedCodes = new[] {HttpStatusCode.Created, HttpStatusCode.Accepted, HttpStatusCode.OK};
837                 if (!expectedCodes.Contains(client.StatusCode))
838                     throw CreateWebException("CreateContainer", client.StatusCode);
839             }
840         }
841
842         public void DeleteContainer(string account, string container)
843         {
844             if (String.IsNullOrWhiteSpace(container))
845                 throw new ArgumentNullException("container", "The container property can't be empty");
846             Contract.EndContractBlock();
847
848             using (var client = new RestClient(_baseClient))
849             {
850                 if (!String.IsNullOrWhiteSpace(account))
851                     client.BaseAddress = GetAccountUrl(account);
852
853                 client.DeleteWithRetry(container, 3);
854                 var expectedCodes = new[] {HttpStatusCode.NotFound, HttpStatusCode.NoContent};
855                 if (!expectedCodes.Contains(client.StatusCode))
856                     throw CreateWebException("DeleteContainer", client.StatusCode);
857             }
858
859         }
860
861         /// <summary>
862         /// 
863         /// </summary>
864         /// <param name="account"></param>
865         /// <param name="container"></param>
866         /// <param name="objectName"></param>
867         /// <param name="fileName"></param>
868         /// <returns></returns>
869         /// <remarks>This method should have no timeout or a very long one</remarks>
870         //Asynchronously download the object specified by *objectName* in a specific *container* to 
871         // a local file
872         public Task GetObject(string account, string container, string objectName, string fileName)
873         {
874             if (String.IsNullOrWhiteSpace(container))
875                 throw new ArgumentNullException("container", "The container property can't be empty");
876             if (String.IsNullOrWhiteSpace(objectName))
877                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");            
878             Contract.EndContractBlock();
879
880             try
881             {
882                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
883                 //object to avoid concurrency errors.
884                 //
885                 //Download operations take a long time therefore they have no timeout.
886                 var client = new RestClient(_baseClient) { Timeout = 0 };
887                 if (!String.IsNullOrWhiteSpace(account))
888                     client.BaseAddress = GetAccountUrl(account);
889
890                 //The container and objectName are relative names. They are joined with the client's
891                 //BaseAddress to create the object's absolute address
892                 var builder = client.GetAddressBuilder(container, objectName);
893                 var uri = builder.Uri;
894
895                 //Download progress is reported to the Trace log
896                 Log.InfoFormat("[GET] START {0}", objectName);
897                 client.DownloadProgressChanged += (sender, args) => 
898                     Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
899                                     fileName, args.ProgressPercentage,
900                                     args.BytesReceived,
901                                     args.TotalBytesToReceive);                                
902
903
904                 //Start downloading the object asynchronously
905                 var downloadTask = client.DownloadFileTask(uri, fileName);
906                 
907                 //Once the download completes
908                 return downloadTask.ContinueWith(download =>
909                                       {
910                                           //Delete the local client object
911                                           client.Dispose();
912                                           //And report failure or completion
913                                           if (download.IsFaulted)
914                                           {
915                                               Log.ErrorFormat("[GET] FAIL for {0} with \r{1}", objectName,
916                                                                download.Exception);
917                                           }
918                                           else
919                                           {
920                                               Log.InfoFormat("[GET] END {0}", objectName);                                             
921                                           }
922                                       });
923             }
924             catch (Exception exc)
925             {
926                 Log.ErrorFormat("[GET] END {0} with {1}", objectName, exc);
927                 throw;
928             }
929
930
931
932         }
933
934         public Task<IList<string>> PutHashMap(string account, string container, string objectName, TreeHash hash)
935         {
936             if (String.IsNullOrWhiteSpace(container))
937                 throw new ArgumentNullException("container");
938             if (String.IsNullOrWhiteSpace(objectName))
939                 throw new ArgumentNullException("objectName");
940             if (hash==null)
941                 throw new ArgumentNullException("hash");
942             if (String.IsNullOrWhiteSpace(Token))
943                 throw new InvalidOperationException("Invalid Token");
944             if (StorageUrl == null)
945                 throw new InvalidOperationException("Invalid Storage Url");
946             Contract.EndContractBlock();
947
948
949             //Don't use a timeout because putting the hashmap may be a long process
950             var client = new RestClient(_baseClient) { Timeout = 0 };           
951             if (!String.IsNullOrWhiteSpace(account))
952                 client.BaseAddress = GetAccountUrl(account);
953
954             //The container and objectName are relative names. They are joined with the client's
955             //BaseAddress to create the object's absolute address
956             var builder = client.GetAddressBuilder(container, objectName);
957             builder.Query = "format=json&hashmap";
958             var uri = builder.Uri;
959
960
961             //Send the tree hash as Json to the server            
962             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
963             var jsonHash = hash.ToJson();
964             var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);
965             
966             return uploadTask.ContinueWith(t =>
967             {
968
969                 var empty = (IList<string>)new List<string>();
970                 
971
972                 //The server will respond either with 201-created if all blocks were already on the server
973                 if (client.StatusCode == HttpStatusCode.Created)                    
974                 {
975                     //in which case we return an empty hash list
976                     return empty;
977                 }
978                 //or with a 409-conflict and return the list of missing parts
979                 //A 409 will cause an exception so we need to check t.IsFaulted to avoid propagating the exception                
980                 if (t.IsFaulted)
981                 {
982                     var ex = t.Exception.InnerException;
983                     var we = ex as WebException;
984                     var response = we.Response as HttpWebResponse;
985                     if (response!=null && response.StatusCode==HttpStatusCode.Conflict)
986                     {
987                         //In case of 409 the missing parts will be in the response content                        
988                         using (var stream = response.GetResponseStream())
989                         using(var reader=new StreamReader(stream))
990                         {
991                             Debug.Assert(stream.Position == 0);
992                             //We need to cleanup the content before returning it because it contains
993                             //error content after the list of hashes
994                             var hashes = new List<string>();
995                             string line=null;
996                             //All lines up to the first empty line are hashes
997                             while(!String.IsNullOrWhiteSpace(line=reader.ReadLine()))
998                             {
999                                 hashes.Add(line);
1000                             }
1001
1002                             return hashes;
1003                         }                        
1004                     }
1005                     else
1006                         //Any other status code is unexpected and the exception should be rethrown
1007                         throw ex;
1008                     
1009                 }
1010                 //Any other status code is unexpected but there was no exception. We can probably continue processing
1011                 else
1012                 {
1013                     Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);                    
1014                 }
1015                 return empty;
1016             });
1017
1018         }
1019
1020         public Task<byte[]> GetBlock(string account, string container, Uri relativeUrl, long start, long? end)
1021         {
1022             if (String.IsNullOrWhiteSpace(Token))
1023                 throw new InvalidOperationException("Invalid Token");
1024             if (StorageUrl == null)
1025                 throw new InvalidOperationException("Invalid Storage Url");
1026             if (String.IsNullOrWhiteSpace(container))
1027                 throw new ArgumentNullException("container");
1028             if (relativeUrl== null)
1029                 throw new ArgumentNullException("relativeUrl");
1030             if (end.HasValue && end<0)
1031                 throw new ArgumentOutOfRangeException("end");
1032             if (start<0)
1033                 throw new ArgumentOutOfRangeException("start");
1034             Contract.EndContractBlock();
1035
1036
1037             //Don't use a timeout because putting the hashmap may be a long process
1038             var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end};
1039             if (!String.IsNullOrWhiteSpace(account))
1040                 client.BaseAddress = GetAccountUrl(account);
1041
1042             var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
1043             var uri = builder.Uri;
1044
1045             return client.DownloadDataTask(uri)
1046                 .ContinueWith(t=>
1047                                   {
1048                                       client.Dispose();
1049                                       return t.Result;
1050                                   });
1051         }
1052
1053
1054         public Task PostBlock(string account, string container, byte[] block, int offset, int count)
1055         {
1056             if (String.IsNullOrWhiteSpace(container))
1057                 throw new ArgumentNullException("container");
1058             if (block == null)
1059                 throw new ArgumentNullException("block");
1060             if (offset < 0 || offset >= block.Length)
1061                 throw new ArgumentOutOfRangeException("offset");
1062             if (count < 0 || count > block.Length)
1063                 throw new ArgumentOutOfRangeException("count");
1064             if (String.IsNullOrWhiteSpace(Token))
1065                 throw new InvalidOperationException("Invalid Token");
1066             if (StorageUrl == null)
1067                 throw new InvalidOperationException("Invalid Storage Url");                        
1068             Contract.EndContractBlock();
1069
1070                         
1071             //Don't use a timeout because putting the hashmap may be a long process
1072             var client = new RestClient(_baseClient) { Timeout = 0 };
1073             if (!String.IsNullOrWhiteSpace(account))
1074                 client.BaseAddress = GetAccountUrl(account);
1075
1076             var builder = client.GetAddressBuilder(container, "");
1077             //We are doing an update
1078             builder.Query = "update";
1079             var uri = builder.Uri;
1080
1081             client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
1082
1083             Log.InfoFormat("[BLOCK POST] START");
1084
1085             client.UploadProgressChanged += (sender, args) => 
1086                 Log.InfoFormat("[BLOCK POST PROGRESS] {0}% {1} of {2}",
1087                                     args.ProgressPercentage, args.BytesSent,
1088                                     args.TotalBytesToSend);
1089             client.UploadFileCompleted += (sender, args) => 
1090                 Log.InfoFormat("[BLOCK POST PROGRESS] Completed ");
1091
1092             
1093             //Send the block
1094             var uploadTask = client.UploadDataTask(uri, "POST", block)
1095             .ContinueWith(upload =>
1096             {
1097                 client.Dispose();
1098
1099                 if (upload.IsFaulted)
1100                 {
1101                     var exception = upload.Exception.InnerException;
1102                     Log.ErrorFormat("[BLOCK POST] FAIL with \r{0}", exception);                        
1103                     throw exception;
1104                 }
1105                     
1106                 Log.InfoFormat("[BLOCK POST] END");
1107             });
1108             return uploadTask;            
1109         }
1110
1111
1112         public Task<TreeHash> GetHashMap(string account, string container, string objectName)
1113         {
1114             if (String.IsNullOrWhiteSpace(container))
1115                 throw new ArgumentNullException("container");
1116             if (String.IsNullOrWhiteSpace(objectName))
1117                 throw new ArgumentNullException("objectName");
1118             if (String.IsNullOrWhiteSpace(Token))
1119                 throw new InvalidOperationException("Invalid Token");
1120             if (StorageUrl == null)
1121                 throw new InvalidOperationException("Invalid Storage Url");
1122             Contract.EndContractBlock();
1123
1124             try
1125             {
1126                 //WebClient, and by extension RestClient, are not thread-safe. Create a new RestClient
1127                 //object to avoid concurrency errors.
1128                 //
1129                 //Download operations take a long time therefore they have no timeout.
1130                 //TODO: Do we really? this is a hashmap operation, not a download
1131                 var client = new RestClient(_baseClient) { Timeout = 0 };
1132                 if (!String.IsNullOrWhiteSpace(account))
1133                     client.BaseAddress = GetAccountUrl(account);
1134
1135
1136                 //The container and objectName are relative names. They are joined with the client's
1137                 //BaseAddress to create the object's absolute address
1138                 var builder = client.GetAddressBuilder(container, objectName);
1139                 builder.Query = "format=json&hashmap";
1140                 var uri = builder.Uri;
1141                 
1142                 //Start downloading the object asynchronously
1143                 var downloadTask = client.DownloadStringTask(uri);
1144                 
1145                 //Once the download completes
1146                 return downloadTask.ContinueWith(download =>
1147                 {
1148                     //Delete the local client object
1149                     client.Dispose();
1150                     //And report failure or completion
1151                     if (download.IsFaulted)
1152                     {
1153                         Log.ErrorFormat("[GET HASH] FAIL for {0} with \r{1}", objectName,
1154                                         download.Exception);
1155                         throw download.Exception;
1156                     }
1157                                           
1158                     //The server will return an empty string if the file is empty
1159                     var json = download.Result;
1160                     var treeHash = TreeHash.Parse(json);
1161                     Log.InfoFormat("[GET HASH] END {0}", objectName);                                             
1162                     return treeHash;
1163                 });
1164             }
1165             catch (Exception exc)
1166             {
1167                 Log.ErrorFormat("[GET HASH] END {0} with {1}", objectName, exc);
1168                 throw;
1169             }
1170
1171
1172
1173         }
1174
1175
1176         /// <summary>
1177         /// 
1178         /// </summary>
1179         /// <param name="account"></param>
1180         /// <param name="container"></param>
1181         /// <param name="objectName"></param>
1182         /// <param name="fileName"></param>
1183         /// <param name="hash">Optional hash value for the file. If no hash is provided, the method calculates a new hash</param>
1184         /// <remarks>>This method should have no timeout or a very long one</remarks>
1185         public Task PutObject(string account, string container, string objectName, string fileName, string hash = null)
1186         {
1187             if (String.IsNullOrWhiteSpace(container))
1188                 throw new ArgumentNullException("container", "The container property can't be empty");
1189             if (String.IsNullOrWhiteSpace(objectName))
1190                 throw new ArgumentNullException("objectName", "The objectName property can't be empty");
1191             if (String.IsNullOrWhiteSpace(fileName))
1192                 throw new ArgumentNullException("fileName", "The fileName property can't be empty");
1193             if (!File.Exists(fileName))
1194                 throw new FileNotFoundException("The file does not exist",fileName);
1195             Contract.EndContractBlock();
1196             
1197             try
1198             {
1199
1200                 var client = new RestClient(_baseClient){Timeout=0};
1201                 if (!String.IsNullOrWhiteSpace(account))
1202                     client.BaseAddress = GetAccountUrl(account);
1203
1204                 var builder = client.GetAddressBuilder(container, objectName);
1205                 var uri = builder.Uri;
1206
1207                 string etag = hash ?? CalculateHash(fileName);
1208
1209                 client.Headers.Add("Content-Type", "application/octet-stream");
1210                 client.Headers.Add("ETag", etag);
1211
1212
1213                 Log.InfoFormat("[PUT] START {0}", objectName);
1214                 client.UploadProgressChanged += (sender, args) =>
1215                 {
1216                     using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
1217                     {
1218                         Log.InfoFormat("{0} {1}% {2} of {3}", fileName, args.ProgressPercentage,
1219                                        args.BytesSent, args.TotalBytesToSend);
1220                     }
1221                 };
1222
1223                 client.UploadFileCompleted += (sender, args) =>
1224                 {
1225                     using (log4net.ThreadContext.Stacks["PUT"].Push("Progress"))
1226                     {
1227                         Log.InfoFormat("Completed {0}", fileName);
1228                     }
1229                 };
1230                 return client.UploadFileTask(uri, "PUT", fileName)
1231                     .ContinueWith(upload=>
1232                                       {
1233                                           client.Dispose();
1234
1235                                           if (upload.IsFaulted)
1236                                           {
1237                                               var exc = upload.Exception.InnerException;
1238                                               Log.ErrorFormat("[PUT] FAIL for {0} with \r{1}",objectName,exc);
1239                                               throw exc;
1240                                           }
1241                                           else
1242                                             Log.InfoFormat("[PUT] END {0}", objectName);
1243                                       });
1244             }
1245             catch (Exception exc)
1246             {
1247                 Log.ErrorFormat("[PUT] END {0} with {1}", objectName, exc);
1248                 throw;
1249             }                
1250
1251         }
1252        
1253         
1254         private static string CalculateHash(string fileName)
1255         {
1256             Contract.Requires(!String.IsNullOrWhiteSpace(fileName));
1257             Contract.EndContractBlock();
1258
1259             string hash;
1260             using (var hasher = MD5.Create())
1261             using(var stream=File.OpenRead(fileName))
1262             {
1263                 var hashBuilder=new StringBuilder();
1264                 foreach (byte b in hasher.ComputeHash(stream))
1265                     hashBuilder.Append(b.ToString("x2").ToLower());
1266                 hash = hashBuilder.ToString();                
1267             }
1268             return hash;
1269         }
1270         
1271         public void MoveObject(string account, string sourceContainer, string oldObjectName, string targetContainer, string newObjectName)
1272         {
1273             if (String.IsNullOrWhiteSpace(sourceContainer))
1274                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1275             if (String.IsNullOrWhiteSpace(oldObjectName))
1276                 throw new ArgumentNullException("oldObjectName", "The oldObjectName property can't be empty");
1277             if (String.IsNullOrWhiteSpace(targetContainer))
1278                 throw new ArgumentNullException("targetContainer", "The container property can't be empty");
1279             if (String.IsNullOrWhiteSpace(newObjectName))
1280                 throw new ArgumentNullException("newObjectName", "The newObjectName property can't be empty");
1281             Contract.EndContractBlock();
1282
1283             var targetUrl = targetContainer + "/" + newObjectName;
1284             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, oldObjectName);
1285
1286             using (var client = new RestClient(_baseClient))
1287             {
1288                 if (!String.IsNullOrWhiteSpace(account))
1289                     client.BaseAddress = GetAccountUrl(account);
1290
1291                 client.Headers.Add("X-Move-From", sourceUrl);
1292                 client.PutWithRetry(targetUrl, 3);
1293
1294                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
1295                 if (!expectedCodes.Contains(client.StatusCode))
1296                     throw CreateWebException("MoveObject", client.StatusCode);
1297             }
1298         }
1299
1300         public void DeleteObject(string account, string sourceContainer, string objectName)
1301         {            
1302             if (String.IsNullOrWhiteSpace(sourceContainer))
1303                 throw new ArgumentNullException("sourceContainer", "The container property can't be empty");
1304             if (String.IsNullOrWhiteSpace(objectName))
1305                 throw new ArgumentNullException("objectName", "The oldObjectName property can't be empty");
1306             Contract.EndContractBlock();
1307
1308             var targetUrl = FolderConstants.TrashContainer + "/" + objectName;
1309             var sourceUrl = String.Format("/{0}/{1}", sourceContainer, objectName);
1310
1311             using (var client = new RestClient(_baseClient))
1312             {
1313                 if (!String.IsNullOrWhiteSpace(account))
1314                     client.BaseAddress = GetAccountUrl(account);
1315
1316                 client.Headers.Add("X-Move-From", sourceUrl);
1317                 client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
1318                 client.PutWithRetry(targetUrl, 3);
1319
1320                 var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
1321                 if (!expectedCodes.Contains(client.StatusCode))
1322                     throw CreateWebException("DeleteObject", client.StatusCode);
1323             }
1324         }
1325
1326       
1327         private static WebException CreateWebException(string operation, HttpStatusCode statusCode)
1328         {
1329             return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
1330         }
1331
1332         
1333     }
1334
1335     public class ShareAccountInfo
1336     {
1337         public DateTime? last_modified { get; set; }
1338         public string name { get; set; }
1339     }
1340 }