This is an automated email from the ASF dual-hosted git repository.
motus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/reef.git
The following commit(s) were added to refs/heads/master by this push:
new 58cd694 [REEF-2035] BlobFs parameters should be account name and key
(#1472)
58cd694 is described below
commit 58cd6946ff9c302ff06699d4086670921afe2140
Author: mandyshieh <[email protected]>
AuthorDate: Mon Jul 9 15:08:50 2018 -0700
[REEF-2035] BlobFs parameters should be account name and key (#1472)
Take out the connection string parameter and replace with account name and
account key. Connection string is then constructed whenever necessary.
JIRA: [REEF-2035] (https://issues.apache.org/jira/browse/REEF-2035)
Pull Request:
This closes #1472
---
.../YARN/HDI/HDInsightClientConfiguration.cs | 2 +-
lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs | 12 +++++++-----
...BlockBlobFileSystem.cs => TestAzureBlobFileSystem.cs} | 7 ++++---
...lobFileSystemE2E.cs => TestAzureBlobFileSystemE2E.cs} | 12 ++++++++----
...zureBlockBlobFileSystem.cs => AzureBlobFileSystem.cs} | 4 ++--
...figuration.cs => AzureBlobFileSystemConfiguration.cs} | 16 +++++++++-------
...er.cs => AzureBlobFileSystemConfigurationProvider.cs} | 14 ++++++++------
.../FileSystem/AzureBlob/AzureCloudBlobClient.cs | 5 ++++-
...ConnectionString.cs => AzureBlobStorageAccountKey.cs} | 6 +++---
...onnectionString.cs => AzureBlobStorageAccountName.cs} | 6 +++---
10 files changed, 49 insertions(+), 35 deletions(-)
diff --git
a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightClientConfiguration.cs
b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightClientConfiguration.cs
index 4b91146..4c64cb9 100644
--- a/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightClientConfiguration.cs
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/HDI/HDInsightClientConfiguration.cs
@@ -76,7 +76,7 @@ namespace Org.Apache.REEF.Client.YARN.HDI
GenericType<HDInsightCommandLineEnvironment>.Class)
.BindImplementation(GenericType<IResourceFileRemoteUrlToClusterUrlConverter>.Class,
GenericType<HDInsightResourceFileRemoteUrlToClusterUrlConverter>.Class)
- .Merge(AzureBlockBlobFileSystemConfiguration.ConfigurationModule)
+ .Merge(AzureBlobFileSystemConfiguration.ConfigurationModule)
.Build();
}
}
\ No newline at end of file
diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
index fede636..4cb1c27 100644
--- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
+++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
@@ -104,16 +104,18 @@ namespace Org.Apache.REEF.Examples.HelloREEF
return
YARNClientConfiguration.ConfigurationModuleYARNRest.Build();
case HDInsight:
// To run against HDInsight please replace placeholders
below, with actual values for
- // connection string, container name (available at Azure
portal) and HDInsight
+ // blob storage account name and key, container name
(available at Azure portal) and HDInsight
// credentials (username and password)
- const string connectionString = "ConnString";
- const string continerName = "foo";
+ const string blobStorageAccountName = "name";
+ const string blobStorageAccountKey = "key";
+ const string containerName = "foo";
return HDInsightClientConfiguration.ConfigurationModule
.Set(HDInsightClientConfiguration.HDInsightPasswordParameter, @"pwd")
.Set(HDInsightClientConfiguration.HDInsightUsernameParameter, @"foo")
.Set(HDInsightClientConfiguration.HDInsightUrlParameter,
@"https://foo.azurehdinsight.net/";)
-
.Set(HDInsightClientConfiguration.JobSubmissionDirectoryPrefix,
string.Format(@"/{0}/tmp", continerName))
-
.Set(AzureBlockBlobFileSystemConfiguration.ConnectionString, connectionString)
+
.Set(HDInsightClientConfiguration.JobSubmissionDirectoryPrefix,
$@"/{containerName}/tmp")
+ .Set(AzureBlobFileSystemConfiguration.AccountName,
blobStorageAccountName)
+ .Set(AzureBlobFileSystemConfiguration.AccountKey,
blobStorageAccountKey)
.Build();
case AzureBatch:
return
AzureBatchRuntimeClientConfiguration.ConfigurationModule
diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystem.cs
b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlobFileSystem.cs
similarity index 95%
rename from lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystem.cs
rename to lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlobFileSystem.cs
index c5b23b9..48ae100 100644
--- a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystem.cs
+++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlobFileSystem.cs
@@ -179,13 +179,14 @@ namespace Org.Apache.REEF.IO.Tests
public IFileSystem GetAzureFileSystem()
{
- var conf =
AzureBlockBlobFileSystemConfiguration.ConfigurationModule
-
.Set(AzureBlockBlobFileSystemConfiguration.ConnectionString,
"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;")
+ var conf = AzureBlobFileSystemConfiguration.ConfigurationModule
+ .Set(AzureBlobFileSystemConfiguration.AccountName,
"accountName")
+ .Set(AzureBlobFileSystemConfiguration.AccountKey,
"accountKey")
.Build();
var injector = TangFactory.GetTang().NewInjector(conf);
injector.BindVolatileInstance(TestCloudBlobClient);
- var fs = injector.GetInstance<AzureBlockBlobFileSystem>();
+ var fs = injector.GetInstance<AzureBlobFileSystem>();
TestCloudBlobClient.BaseUri.ReturnsForAnyArgs(BaseUri);
TestCloudBlockBlob.Open().Returns(TestOpenStream);
TestCloudBlockBlob.Create().Returns(TestCreateStream);
diff --git
a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs
b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlobFileSystemE2E.cs
similarity index 95%
rename from lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs
rename to lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlobFileSystemE2E.cs
index f866944..7ee1268 100644
--- a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs
+++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlobFileSystemE2E.cs
@@ -40,17 +40,21 @@ namespace Org.Apache.REEF.IO.Tests
private IFileSystem _fileSystem;
private CloudBlobClient _client;
private CloudBlobContainer _container;
+ private const string AzureBlobConnectionFormat =
"DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1};";
public TestAzureBlockBlobFileSystemE2E()
{
// Fill in before running test!
- const string ConnectionString =
"DefaultEndpointsProtocol=https;AccountName=myAccount;AccountKey=myKey;EndpointSuffix=core.windows.net";
+ const string AccountName = "";
+ const string AccountKey = "";
+ string ConnectionString = string.Format(AzureBlobConnectionFormat,
AccountName, AccountKey);
var defaultContainerName = "reef-test-container-" + Guid.NewGuid();
- var conf =
AzureBlockBlobFileSystemConfiguration.ConfigurationModule
- .Set(AzureBlockBlobFileSystemConfiguration.ConnectionString,
ConnectionString)
+ var conf = AzureBlobFileSystemConfiguration.ConfigurationModule
+ .Set(AzureBlobFileSystemConfiguration.AccountName, AccountName)
+ .Set(AzureBlobFileSystemConfiguration.AccountKey, AccountKey)
.Build();
- _fileSystem =
TangFactory.GetTang().NewInjector(conf).GetInstance<AzureBlockBlobFileSystem>();
+ _fileSystem =
TangFactory.GetTang().NewInjector(conf).GetInstance<AzureBlobFileSystem>();
_client =
CloudStorageAccount.Parse(ConnectionString).CreateCloudBlobClient();
_container = _client.GetContainerReference(defaultContainerName);
_container.CreateIfNotExists();
diff --git
a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs
b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlobFileSystem.cs
similarity index 98%
rename from
lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs
rename to lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlobFileSystem.cs
index d1004c3..7ff2edc 100644
---
a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs
+++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlobFileSystem.cs
@@ -31,14 +31,14 @@ namespace Org.Apache.REEF.IO.FileSystem.AzureBlob
/// This particular File System implementation only supports
/// block blob operations
/// </summary>
- internal sealed class AzureBlockBlobFileSystem : IFileSystem
+ internal sealed class AzureBlobFileSystem : IFileSystem
{
private readonly ICloudBlobClient _client;
private const char UrlPathSeparator = '/';
[Inject]
- private AzureBlockBlobFileSystem(ICloudBlobClient client)
+ private AzureBlobFileSystem(ICloudBlobClient client)
{
_client = client;
}
diff --git
a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystemConfiguration.cs
b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlobFileSystemConfiguration.cs
similarity index 75%
rename from
lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystemConfiguration.cs
rename to
lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlobFileSystemConfiguration.cs
index 0b25862..98c2477 100644
---
a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystemConfiguration.cs
+++
b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlobFileSystemConfiguration.cs
@@ -30,9 +30,10 @@ namespace Org.Apache.REEF.IO.FileSystem.AzureBlob
/// <remarks>
/// Stream-based operations are not supported.
/// </remarks>
- public sealed class AzureBlockBlobFileSystemConfiguration :
ConfigurationModuleBuilder
+ public sealed class AzureBlobFileSystemConfiguration :
ConfigurationModuleBuilder
{
- public static readonly RequiredParameter<string> ConnectionString =
new RequiredParameter<string>();
+ public static readonly RequiredParameter<string> AccountName = new
RequiredParameter<string>();
+ public static readonly RequiredParameter<string> AccountKey = new
RequiredParameter<string>();
public static readonly OptionalImpl<IAzureBlobRetryPolicy> RetryPolicy
= new OptionalImpl<IAzureBlobRetryPolicy>();
@@ -40,11 +41,12 @@ namespace Org.Apache.REEF.IO.FileSystem.AzureBlob
/// Set AzureBlockBlobFileSystemProvider to
DriverConfigurationProviders.
/// Set all the parameters needed for injecting
AzureBlockBlobFileSystemProvider.
/// </summary>
- public static readonly ConfigurationModule ConfigurationModule = new
AzureBlockBlobFileSystemConfiguration()
- .BindSetEntry<DriverConfigurationProviders,
AzureBlockBlobFileSystemConfigurationProvider, IConfigurationProvider>(
- GenericType<DriverConfigurationProviders>.Class,
GenericType<AzureBlockBlobFileSystemConfigurationProvider>.Class)
- .BindImplementation(GenericType<IFileSystem>.Class,
GenericType<AzureBlockBlobFileSystem>.Class)
-
.BindNamedParameter(GenericType<AzureStorageConnectionString>.Class,
ConnectionString)
+ public static readonly ConfigurationModule ConfigurationModule = new
AzureBlobFileSystemConfiguration()
+ .BindSetEntry<DriverConfigurationProviders,
AzureBlobFileSystemConfigurationProvider, IConfigurationProvider>(
+ GenericType<DriverConfigurationProviders>.Class,
GenericType<AzureBlobFileSystemConfigurationProvider>.Class)
+ .BindImplementation(GenericType<IFileSystem>.Class,
GenericType<AzureBlobFileSystem>.Class)
+
.BindNamedParameter(GenericType<AzureBlobStorageAccountName>.Class, AccountName)
+ .BindNamedParameter(GenericType<AzureBlobStorageAccountKey>.Class,
AccountKey)
.BindImplementation(GenericType<IAzureBlobRetryPolicy>.Class,
RetryPolicy)
.Build();
}
diff --git
a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystemConfigurationProvider.cs
b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlobFileSystemConfigurationProvider.cs
similarity index 78%
rename from
lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystemConfigurationProvider.cs
rename to
lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlobFileSystemConfigurationProvider.cs
index de490c0..a635c6f 100644
---
a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystemConfigurationProvider.cs
+++
b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlobFileSystemConfigurationProvider.cs
@@ -30,20 +30,22 @@ namespace Org.Apache.REEF.IO.FileSystem.AzureBlob
/// The client that is going to use AzureBlockBlobFileSystem in its driver
and evaluators should set
/// configuration data through AzureBlockBlobFileSystem configuration
module in the client's configuration
/// </summary>
- internal sealed class AzureBlockBlobFileSystemConfigurationProvider :
IConfigurationProvider
+ internal sealed class AzureBlobFileSystemConfigurationProvider :
IConfigurationProvider
{
private readonly IConfiguration _configuration;
[Inject]
- private AzureBlockBlobFileSystemConfigurationProvider(
- [Parameter(typeof(AzureStorageConnectionString))] string
connectionString,
+ private AzureBlobFileSystemConfigurationProvider(
+ [Parameter(typeof(AzureBlobStorageAccountName))] string
accountName,
+ [Parameter(typeof(AzureBlobStorageAccountKey))] string accountKey,
IAzureBlobRetryPolicy retryPolicy)
{
_configuration = TangFactory.GetTang().NewConfigurationBuilder()
- .BindImplementation(GenericType<IFileSystem>.Class,
GenericType<AzureBlockBlobFileSystem>.Class)
+ .BindImplementation(GenericType<IFileSystem>.Class,
GenericType<AzureBlobFileSystem>.Class)
.BindImplementation(typeof(IAzureBlobRetryPolicy),
retryPolicy.GetType())
-
.BindStringNamedParam<AzureStorageConnectionString>(connectionString)
- .BindSetEntry<EvaluatorConfigurationProviders,
AzureBlockBlobFileSystemConfigurationProvider, IConfigurationProvider>()
+ .BindStringNamedParam<AzureBlobStorageAccountName>(accountName)
+ .BindStringNamedParam<AzureBlobStorageAccountKey>(accountKey)
+ .BindSetEntry<EvaluatorConfigurationProviders,
AzureBlobFileSystemConfigurationProvider, IConfigurationProvider>()
.Build();
}
diff --git
a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs
b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs
index eec4806..7b0a56a 100644
--- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs
+++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs
@@ -32,6 +32,7 @@ namespace Org.Apache.REEF.IO.FileSystem.AzureBlob
internal sealed class AzureCloudBlobClient : ICloudBlobClient
{
private readonly CloudBlobClient _client;
+ private const string AzureBlobConnectionFormat =
"DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1};";
private readonly BlobRequestOptions _requestOptions;
public StorageCredentials Credentials
@@ -40,9 +41,11 @@ namespace Org.Apache.REEF.IO.FileSystem.AzureBlob
}
[Inject]
- private
AzureCloudBlobClient([Parameter(typeof(AzureStorageConnectionString))] string
connectionString,
+ private
AzureCloudBlobClient([Parameter(typeof(AzureBlobStorageAccountName))] string
accountName,
+
[Parameter(typeof(AzureBlobStorageAccountKey))] string accountKey,
IAzureBlobRetryPolicy retryPolicy)
{
+ var connectionString = string.Format(AzureBlobConnectionFormat,
accountName, accountKey);
_client =
CloudStorageAccount.Parse(connectionString).CreateCloudBlobClient();
_client.DefaultRequestOptions.RetryPolicy = retryPolicy;
_requestOptions = new BlobRequestOptions() { RetryPolicy =
retryPolicy };
diff --git
a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/Parameters/AzureStorageConnectionString.cs
b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/Parameters/AzureBlobStorageAccountKey.cs
similarity index 84%
copy from
lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/Parameters/AzureStorageConnectionString.cs
copy to
lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/Parameters/AzureBlobStorageAccountKey.cs
index c7e0540..f59cf42 100644
---
a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/Parameters/AzureStorageConnectionString.cs
+++
b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/Parameters/AzureBlobStorageAccountKey.cs
@@ -19,10 +19,10 @@ using Org.Apache.REEF.Tang.Annotations;
namespace Org.Apache.REEF.IO.FileSystem.AzureBlob.Parameters
{
- [NamedParameter("The connection string for Azure Storage")]
- public sealed class AzureStorageConnectionString : Name<string>
+ [NamedParameter("The Azure Blob Storage account key")]
+ public sealed class AzureBlobStorageAccountKey : Name<string>
{
- private AzureStorageConnectionString()
+ private AzureBlobStorageAccountKey()
{
}
}
diff --git
a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/Parameters/AzureStorageConnectionString.cs
b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/Parameters/AzureBlobStorageAccountName.cs
similarity index 84%
rename from
lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/Parameters/AzureStorageConnectionString.cs
rename to
lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/Parameters/AzureBlobStorageAccountName.cs
index c7e0540..0556869 100644
---
a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/Parameters/AzureStorageConnectionString.cs
+++
b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/Parameters/AzureBlobStorageAccountName.cs
@@ -19,10 +19,10 @@ using Org.Apache.REEF.Tang.Annotations;
namespace Org.Apache.REEF.IO.FileSystem.AzureBlob.Parameters
{
- [NamedParameter("The connection string for Azure Storage")]
- public sealed class AzureStorageConnectionString : Name<string>
+ [NamedParameter("The Azure Blob Storage account name")]
+ public sealed class AzureBlobStorageAccountName : Name<string>
{
- private AzureStorageConnectionString()
+ private AzureBlobStorageAccountName()
{
}
}
