X-Git-Url: https://code.grnet.gr/git/pithos-ms-client/blobdiff_plain/f3d080df476f28bb731d09efce2e1538b1fbb420..97edb52fa8c0dbe6b4856c3972b13436907b4b25:/trunk/Pithos.Network/CloudFilesClient.cs
diff --git a/trunk/Pithos.Network/CloudFilesClient.cs b/trunk/Pithos.Network/CloudFilesClient.cs
index 0cb2355..c4496c9 100644
--- a/trunk/Pithos.Network/CloudFilesClient.cs
+++ b/trunk/Pithos.Network/CloudFilesClient.cs
@@ -1,40 +1,44 @@
-// -----------------------------------------------------------------------
-//
-// Copyright 2011 GRNET S.A. All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or
-// without modification, are permitted provided that the following
-// conditions are met:
-//
-// 1. Redistributions of source code must retain the above
-// copyright notice, this list of conditions and the following
-// disclaimer.
-//
-// 2. Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following
-// disclaimer in the documentation and/or other materials
-// provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
-// OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
-// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-// USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
-// AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-// POSSIBILITY OF SUCH DAMAGE.
-//
-// The views and conclusions contained in the software and
-// documentation are those of the authors and should not be
-// interpreted as representing official policies, either expressed
-// or implied, of GRNET S.A.
-//
-// -----------------------------------------------------------------------
-
+#region
+/* -----------------------------------------------------------------------
+ *
+ *
+ * Copyright 2011-2012 GRNET S.A. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ *
+ * THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and
+ * documentation are those of the authors and should not be
+ * interpreted as representing official policies, either expressed
+ * or implied, of GRNET S.A.
+ *
+ * -----------------------------------------------------------------------
+ */
+#endregion
// **CloudFilesClient** provides a simple client interface to CloudFiles and Pithos
//
@@ -50,8 +54,10 @@ using System.Diagnostics.Contracts;
using System.IO;
using System.Linq;
using System.Net;
+using System.Reflection;
using System.Security.Cryptography;
using System.Text;
+using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Pithos.Interfaces;
@@ -62,6 +68,8 @@ namespace Pithos.Network
[Export(typeof(ICloudClient))]
public class CloudFilesClient:ICloudClient
{
+ private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+
//CloudFilesClient uses *_baseClient* internally to communicate with the server
//RestClient provides a REST-friendly interface over the standard WebClient.
private RestClient _baseClient;
@@ -93,7 +101,20 @@ namespace Pithos.Network
protected Uri RootAddressUri { get; set; }
- private Uri _proxy;
+ /* private WebProxy _proxy;
+ public WebProxy Proxy
+ {
+ get { return _proxy; }
+ set
+ {
+ _proxy = value;
+ if (_baseClient != null)
+ _baseClient.Proxy = value;
+ }
+ }
+*/
+
+ /* private Uri _proxy;
public Uri Proxy
{
get { return _proxy; }
@@ -103,7 +124,7 @@ namespace Pithos.Network
if (_baseClient != null)
_baseClient.Proxy = new WebProxy(value);
}
- }
+ }*/
public double DownloadPercentLimit { get; set; }
public double UploadPercentLimit { get; set; }
@@ -119,7 +140,6 @@ namespace Pithos.Network
public bool UsePithos { get; set; }
- private static readonly ILog Log = LogManager.GetLogger("CloudFilesClient");
public CloudFilesClient(string userName, string apiKey)
{
@@ -135,13 +155,13 @@ namespace Pithos.Network
Contract.Ensures(StorageUrl != null);
Contract.Ensures(_baseClient != null);
Contract.Ensures(RootAddressUri != null);
- Contract.EndContractBlock();
+ Contract.EndContractBlock();
_baseClient = new RestClient
{
BaseAddress = accountInfo.StorageUri.ToString(),
Timeout = 10000,
- Retries = 3
+ Retries = 3,
};
StorageUrl = accountInfo.StorageUri;
Token = accountInfo.Token;
@@ -175,9 +195,9 @@ namespace Pithos.Network
var groups = new List();
using (var authClient = new RestClient{BaseAddress=AuthenticationUrl})
- {
- if (Proxy != null)
- authClient.Proxy = new WebProxy(Proxy);
+ {
+ /* if (Proxy != null)
+ authClient.Proxy = Proxy;*/
Contract.Assume(authClient.Headers!=null);
@@ -196,7 +216,8 @@ namespace Pithos.Network
{
BaseAddress = storageUrl,
Timeout = 10000,
- Retries = 3
+ Retries = 3,
+ //Proxy=Proxy
};
StorageUrl = new Uri(storageUrl);
@@ -222,7 +243,7 @@ namespace Pithos.Network
}
Log.InfoFormat("[AUTHENTICATE] End for {0}", UserName);
-
+ Debug.Assert(_baseClient!=null);
return new AccountInfo {StorageUri = StorageUrl, Token = Token, UserName = UserName,Groups=groups};
@@ -299,18 +320,21 @@ namespace Pithos.Network
using (ThreadContext.Stacks["Share"].Push("List Objects"))
{
if (Log.IsDebugEnabled) Log.DebugFormat("START");
-
+ //'since' is not used here because we need to have ListObjects return a NoChange result
+ //for all shared accounts,containers
+ var accounts = ListSharingAccounts();
+ var items = from account in accounts
+ let containers = ListContainers(account.name)
+ from container in containers
+ select ListObjects(account.name, container.Name,since);
+ var objects=items.SelectMany(r=> r).ToList();
+/*
var objects = new List();
- var accounts = ListSharingAccounts(since);
- foreach (var account in accounts)
+ foreach (var containerObjects in items)
{
- var containers = ListContainers(account.name);
- foreach (var container in containers)
- {
- var containerObjects = ListObjects(account.name, container.Name);
- objects.AddRange(containerObjects);
- }
+ objects.AddRange(containerObjects);
}
+*/
if (Log.IsDebugEnabled) Log.DebugFormat("END");
return objects;
}
@@ -487,10 +511,13 @@ namespace Pithos.Network
client.Headers.Add("X-Object-Public", isPublic);
- var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);
- var uri = uriBuilder.Uri;
-
- client.UploadValues(uri,new NameValueCollection());
+ /*var uriBuilder = client.GetAddressBuilder(objectInfo.Container, objectInfo.Name);
+ uriBuilder.Query = "update=";
+ var uri = uriBuilder.Uri.MakeRelativeUri(this.RootAddressUri);*/
+ var address = String.Format("{0}/{1}?update=",objectInfo.Container, objectInfo.Name);
+ client.PostWithRetry(address,"application/xml");
+
+ //client.UploadValues(uri,new NameValueCollection());
client.AssertStatusOK("UpdateMetadata failed");
@@ -585,6 +612,8 @@ namespace Pithos.Network
client.AssertStatusOK("ListObjects failed");
+ if (client.StatusCode==HttpStatusCode.NotModified)
+ return new[]{new NoModificationInfo(account,container)};
//If the result is empty, return an empty list,
var infos = String.IsNullOrWhiteSpace(content)
? new List()
@@ -595,21 +624,22 @@ namespace Pithos.Network
{
info.Container = container;
info.Account = account;
+ info.StorageUri = this.StorageUrl;
}
- if (Log.IsDebugEnabled) Log.DebugFormat("START");
+ if (Log.IsDebugEnabled) Log.DebugFormat("END");
return infos;
}
}
}
-
-
public IList ListObjects(string account, string container, string folder, DateTime? since = null)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container");
+/*
if (String.IsNullOrWhiteSpace(folder))
throw new ArgumentNullException("folder");
+*/
Contract.EndContractBlock();
using (ThreadContext.Stacks["Objects"].Push("List"))
@@ -628,8 +658,17 @@ namespace Pithos.Network
var content = client.DownloadStringWithRetry(container, 3);
client.AssertStatusOK("ListObjects failed");
- var infos = JsonConvert.DeserializeObject>(content);
+ if (client.StatusCode==HttpStatusCode.NotModified)
+ return new[]{new NoModificationInfo(account,container,folder)};
+ var infos = JsonConvert.DeserializeObject>(content);
+ foreach (var info in infos)
+ {
+ info.Account = account;
+ if (info.Container == null)
+ info.Container = container;
+ info.StorageUri = this.StorageUrl;
+ }
if (Log.IsDebugEnabled) Log.DebugFormat("END");
return infos;
}
@@ -749,7 +788,8 @@ namespace Pithos.Network
Account = account,
Container = container,
Name = objectName,
- Hash = client.GetHeaderValue("ETag"),
+ ETag = client.GetHeaderValue("ETag"),
+ X_Object_Hash = client.GetHeaderValue("X-Object-Hash"),
Content_Type = client.GetHeaderValue("Content-Type"),
Bytes = Convert.ToInt64(client.GetHeaderValue("Content-Length",true)),
Tags = tags,
@@ -758,7 +798,8 @@ namespace Pithos.Network
ContentEncoding=client.GetHeaderValue("Content-Encoding",true),
ContendDisposition = client.GetHeaderValue("Content-Disposition",true),
Manifest=client.GetHeaderValue("X-Object-Manifest",true),
- PublicUrl=client.GetHeaderValue("X-Object-Public",true),
+ PublicUrl=client.GetHeaderValue("X-Object-Public",true),
+ StorageUri=this.StorageUrl,
};
info.SetPermissions(permissions);
return info;
@@ -837,6 +878,7 @@ namespace Pithos.Network
{
Account=account,
Name = container,
+ StorageUrl=this.StorageUrl.ToString(),
Count =
long.Parse(client.GetHeaderValue("X-Container-Object-Count")),
Bytes = long.Parse(client.GetHeaderValue("X-Container-Bytes-Used")),
@@ -907,7 +949,7 @@ namespace Pithos.Network
/// This method should have no timeout or a very long one
//Asynchronously download the object specified by *objectName* in a specific *container* to
// a local file
- public Task GetObject(string account, string container, string objectName, string fileName)
+ public async Task GetObject(string account, string container, string objectName, string fileName,CancellationToken cancellationToken)
{
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container", "The container property can't be empty");
@@ -921,50 +963,40 @@ namespace Pithos.Network
//object to avoid concurrency errors.
//
//Download operations take a long time therefore they have no timeout.
- var client = new RestClient(_baseClient) { Timeout = 0 };
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
+ using(var client = new RestClient(_baseClient) { Timeout = 0 })
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
- //The container and objectName are relative names. They are joined with the client's
- //BaseAddress to create the object's absolute address
- var builder = client.GetAddressBuilder(container, objectName);
- var uri = builder.Uri;
+ //The container and objectName are relative names. They are joined with the client's
+ //BaseAddress to create the object's absolute address
+ var builder = client.GetAddressBuilder(container, objectName);
+ var uri = builder.Uri;
- //Download progress is reported to the Trace log
- Log.InfoFormat("[GET] START {0}", objectName);
- client.DownloadProgressChanged += (sender, args) =>
- Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
- fileName, args.ProgressPercentage,
- args.BytesReceived,
- args.TotalBytesToReceive);
+ //Download progress is reported to the Trace log
+ Log.InfoFormat("[GET] START {0}", objectName);
+ client.DownloadProgressChanged += (sender, args) =>
+ Log.InfoFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
+ fileName, args.ProgressPercentage,
+ args.BytesReceived,
+ args.TotalBytesToReceive);
+
+ //Start downloading the object asynchronously
+ await client.DownloadFileTaskAsync(uri, fileName,cancellationToken);
- //Start downloading the object asynchronously
- var downloadTask = client.DownloadFileTask(uri, fileName);
-
- //Once the download completes
- return downloadTask.ContinueWith(download =>
- {
- //Delete the local client object
- client.Dispose();
- //And report failure or completion
- if (download.IsFaulted)
- {
- Log.ErrorFormat("[GET] FAIL for {0} with \r{1}", objectName,
- download.Exception);
- }
- else
- {
- Log.InfoFormat("[GET] END {0}", objectName);
- }
- });
+ //Once the download completes
+ //Delete the local client object
+ }
+ //And report failure or completion
}
catch (Exception exc)
{
- Log.ErrorFormat("[GET] END {0} with {1}", objectName, exc);
+ Log.ErrorFormat("[GET] FAIL {0} with {1}", objectName, exc);
throw;
}
+ Log.InfoFormat("[GET] END {0}", objectName);
}
@@ -1000,7 +1032,8 @@ namespace Pithos.Network
client.Headers[HttpRequestHeader.ContentType] = "application/octet-stream";
var jsonHash = hash.ToJson();
var uploadTask=client.UploadStringTask(uri, "PUT", jsonHash);
-
+ if (Log.IsDebugEnabled)
+ Log.DebugFormat("Hashes:\r\n{0}", jsonHash);
return uploadTask.ContinueWith(t =>
{
@@ -1024,26 +1057,26 @@ namespace Pithos.Network
{
//In case of 409 the missing parts will be in the response content
using (var stream = response.GetResponseStream())
- using(var reader=new StreamReader(stream))
+ using(var reader=stream.GetLoggedReader(Log))
{
- Debug.Assert(stream.Position == 0);
- //We need to cleanup the content before returning it because it contains
+ //We used to have to cleanup the content before returning it because it contains
//error content after the list of hashes
- var hashes = new List();
- string line=null;
- //All lines up to the first empty line are hashes
- while(!String.IsNullOrWhiteSpace(line=reader.ReadLine()))
- {
- hashes.Add(line);
- }
-
+ //
+ //As of 30/1/2012, the result is a proper Json array so we don't need to read the content
+ //line by line
+
+ var serializer = new JsonSerializer();
+ serializer.Error += (sender, args) => Log.ErrorFormat("Deserialization error at [{0}] [{1}]", args.ErrorContext.Error, args.ErrorContext.Member);
+ var hashes = (List)serializer.Deserialize(reader, typeof(List));
return hashes;
}
}
//Any other status code is unexpected and the exception should be rethrown
+ Log.LogError(response);
throw ex;
}
+
//Any other status code is unexpected but there was no exception. We can probably continue processing
Log.WarnFormat("Unexcpected status code when putting map: {0} - {1}",client.StatusCode,client.StatusDescription);
@@ -1052,7 +1085,8 @@ namespace Pithos.Network
}
- public Task GetBlock(string account, string container, Uri relativeUrl, long start, long? end)
+
+ public async Task GetBlock(string account, string container, Uri relativeUrl, long start, long? end, CancellationToken cancellationToken)
{
if (String.IsNullOrWhiteSpace(Token))
throw new InvalidOperationException("Invalid Token");
@@ -1060,29 +1094,33 @@ namespace Pithos.Network
throw new InvalidOperationException("Invalid Storage Url");
if (String.IsNullOrWhiteSpace(container))
throw new ArgumentNullException("container");
- if (relativeUrl== null)
+ if (relativeUrl == null)
throw new ArgumentNullException("relativeUrl");
- if (end.HasValue && end<0)
+ if (end.HasValue && end < 0)
throw new ArgumentOutOfRangeException("end");
- if (start<0)
+ if (start < 0)
throw new ArgumentOutOfRangeException("start");
Contract.EndContractBlock();
-
//Don't use a timeout because putting the hashmap may be a long process
- var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end};
- if (!String.IsNullOrWhiteSpace(account))
- client.BaseAddress = GetAccountUrl(account);
+ using (var client = new RestClient(_baseClient) {Timeout = 0, RangeFrom = start, RangeTo = end})
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+ var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
+ var uri = builder.Uri;
+
+ client.DownloadProgressChanged += (sender, args) =>
+ Log.DebugFormat("[GET PROGRESS] {0} {1}% {2} of {3}",
+ uri.Segments.Last(), args.ProgressPercentage,
+ args.BytesReceived,
+ args.TotalBytesToReceive);
- var builder = client.GetAddressBuilder(container, relativeUrl.ToString());
- var uri = builder.Uri;
- return client.DownloadDataTask(uri)
- .ContinueWith(t=>
- {
- client.Dispose();
- return t.Result;
- });
+ var result = await client.DownloadDataTaskAsync(uri, cancellationToken);
+ return result;
+ }
}
@@ -1108,7 +1146,7 @@ namespace Pithos.Network
//Don't use a timeout because putting the hashmap may be a long process
using (var client = new RestClient(_baseClient) { Timeout = 0 })
- {
+ {
if (!String.IsNullOrWhiteSpace(account))
client.BaseAddress = GetAccountUrl(account);
@@ -1131,7 +1169,7 @@ namespace Pithos.Network
var buffer = new byte[count];
Buffer.BlockCopy(block, offset, buffer, 0, count);
//Send the block
- await client.UploadDataTask(uri, "POST", buffer);
+ await client.UploadDataTaskAsync(uri, "POST", buffer);
Log.InfoFormat("[BLOCK POST] END");
}
}
@@ -1212,7 +1250,6 @@ namespace Pithos.Network
if (!File.Exists(fileName) && !Directory.Exists(fileName))
throw new FileNotFoundException("The file or directory does not exist",fileName);
*/
- Contract.EndContractBlock();
try
{
@@ -1248,7 +1285,7 @@ namespace Pithos.Network
{
Log.InfoFormat("Completed {0}", fileName);
}
- };
+ };
if (contentType=="application/directory")
await client.UploadDataTaskAsync(uri, "PUT", new byte[0]);
else
@@ -1330,6 +1367,7 @@ namespace Pithos.Network
client.Headers.Add("X-Move-From", sourceUrl);
client.AllowedStatusCodes.Add(HttpStatusCode.NotFound);
+ Log.InfoFormat("[TRASH] [{0}] to [{1}]",sourceUrl,targetUrl);
client.PutWithRetry(targetUrl, 3);
var expectedCodes = new[] {HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created,HttpStatusCode.NotFound};
@@ -1344,7 +1382,46 @@ namespace Pithos.Network
return new WebException(String.Format("{0} failed with unexpected status code {1}", operation, statusCode));
}
-
+
+/*
+ public IEnumerable ListDirectories(ContainerInfo container)
+ {
+ var directories=this.ListObjects(container.Account, container.Name, "/");
+ }
+*/
+
+ public bool CanUpload(string account, ObjectInfo cloudFile)
+ {
+ Contract.Requires(!String.IsNullOrWhiteSpace(account));
+ Contract.Requires(cloudFile!=null);
+
+ using (var client = new RestClient(_baseClient))
+ {
+ if (!String.IsNullOrWhiteSpace(account))
+ client.BaseAddress = GetAccountUrl(account);
+
+
+ var parts = cloudFile.Name.Split('/');
+ var folder = String.Join("/", parts,0,parts.Length-1);
+
+ var fileUrl=String.Format("{0}/{1}/{2}.pithos.ignore",cloudFile.Container,folder,Guid.NewGuid());
+
+ client.Parameters.Clear();
+ try
+ {
+ client.PutWithRetry(fileUrl, 3, @"application/octet-stream");
+
+ var expectedCodes = new[] { HttpStatusCode.OK, HttpStatusCode.NoContent, HttpStatusCode.Created};
+ var result=(expectedCodes.Contains(client.StatusCode));
+ DeleteObject(account, cloudFile.Container, fileUrl);
+ return result;
+ }
+ catch
+ {
+ return false;
+ }
+ }
+ }
}
public class ShareAccountInfo