Repository: reef Updated Branches: refs/heads/master 7e1ec477a -> ec738ad33
[REEF-2031] Azure Batch credentials exposed in job-submission-params.json in REEF.NET When running REEF .NET on Azure Batch, Azure Batch key and Azure Storage account key are exposed in job-submission-params.json file. Those keys are unnecessary as environment variables "AZ_BATCH_AUTHENTICATION_TOKEN" is set for Azure Batch token, and "AZURE_STORAGE_CONTAINER_SAS_TOKEN_ENV" is set for Azure Storage Account token. JIRA: [REEF-2031](httaps://issues.apache.org/jira/browse/REEF-2031) Pull request: This closes #1469 Project: http://git-wip-us.apache.org/repos/asf/reef/repo Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/ec738ad3 Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/ec738ad3 Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/ec738ad3 Branch: refs/heads/master Commit: ec738ad33ed0e71872e0699c668ad87116cf3bd8 Parents: 7e1ec47 Author: Chenxi Zhao <33109414+chzh...@users.noreply.github.com> Authored: Tue Jun 5 13:20:57 2018 -0700 Committer: Sergiy Matusevych <mo...@apache.org> Committed: Wed Jun 13 18:23:56 2018 -0700 ---------------------------------------------------------------------- .../AvroAzureBatchJobSubmissionParameters.cs | 20 +------ .../AzureBatch/Util/JobJarMaker.cs | 5 -- .../src/main/avro/JobSubmissionParameters.avsc | 5 +- ...zureBatchBootstrapDriverConfigGenerator.java | 35 ++----------- .../client/AzureBatchBootstrapREEFLauncher.java | 55 +++++++++++++------- 5 files changed, 41 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/reef/blob/ec738ad3/lang/cs/Org.Apache.REEF.Client/Avro/AzureBatch/AvroAzureBatchJobSubmissionParameters.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/Avro/AzureBatch/AvroAzureBatchJobSubmissionParameters.cs b/lang/cs/Org.Apache.REEF.Client/Avro/AzureBatch/AvroAzureBatchJobSubmissionParameters.cs index 78bfe4a..6409dc8 100644 --- a/lang/cs/Org.Apache.REEF.Client/Avro/AzureBatch/AvroAzureBatchJobSubmissionParameters.cs +++ b/lang/cs/Org.Apache.REEF.Client/Avro/AzureBatch/AvroAzureBatchJobSubmissionParameters.cs @@ -30,7 +30,7 @@ namespace Org.Apache.REEF.Client.Avro.AzureBatch [DataContract(Namespace = "org.apache.reef.reef.bridge.client.avro")] public sealed class AvroAzureBatchJobSubmissionParameters { - private const string JsonSchema = @"{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters"",""doc"":""Job submission parameters used by the Azure Batch runtime"",""fields"":[{""name"":""sharedJobSubmissionParameters"",""type"":{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroJobSubmissionParameters"",""doc"":""General cross-language job submission parameters shared by all runtimes"",""fields"":[{""name"":""jobId"",""type"":""string""},{""name"":""jobSubmissionFolder"",""type"":""string""}]}},{""name"":""AzureBatchAccountKey"",""type"":""string""},{""name"":""AzureBatchAccountName"",""type"":""string""},{""name"":""AzureBatchAccountUri"",""type"":""string""},{""name"":""AzureBatchPoolId"",""type"":""string""},{""name"":""AzureStorageAccountKey"",""type"":""string""},{""name"":""AzureStorageAccountName"",""type"":""string""},{""name"":""AzureStorageContainerName"",""type"":""string""},{""name"": ""AzureBatchIsWindows"",""type"":""boolean""}]}"; + private const string JsonSchema = @"{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters"",""doc"":""Job submission parameters used by the Azure Batch runtime"",""fields"":[{""name"":""sharedJobSubmissionParameters"",""type"":{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroJobSubmissionParameters"",""doc"":""General cross-language job submission parameters shared by all runtimes"",""fields"":[{""name"":""jobId"",""type"":""string""},{""name"":""jobSubmissionFolder"",""type"":""string""}]}},{""name"":""AzureBatchAccountName"",""type"":""string""},{""name"":""AzureBatchAccountUri"",""type"":""string""},{""name"":""AzureBatchPoolId"",""type"":""string""},{""name"":""AzureStorageAccountName"",""type"":""string""},{""name"":""AzureStorageContainerName"",""type"":""string""}]}"; /// <summary> /// Gets the schema. @@ -50,12 +50,6 @@ namespace Org.Apache.REEF.Client.Avro.AzureBatch public AvroJobSubmissionParameters sharedJobSubmissionParameters { get; set; } /// <summary> - /// Gets or sets the AzureBatchAccountKey field. - /// </summary> - [DataMember] - public string AzureBatchAccountKey { get; set; } - - /// <summary> /// Gets or sets the AzureBatchAccountName field. /// </summary> [DataMember] @@ -74,12 +68,6 @@ namespace Org.Apache.REEF.Client.Avro.AzureBatch public string AzureBatchPoolId { get; set; } /// <summary> - /// Gets or sets the AzureStorageAccountKey field. - /// </summary> - [DataMember] - public string AzureStorageAccountKey { get; set; } - - /// <summary> /// Gets or sets the AzureStorageAccountName field. /// </summary> [DataMember] @@ -92,12 +80,6 @@ namespace Org.Apache.REEF.Client.Avro.AzureBatch public string AzureStorageContainerName { get; set; } /// <summary> - /// Gets or sets the AzureBatchIsWindows field. - /// </summary> - [DataMember] - public bool AzureBatchIsWindows { get; set; } - - /// <summary> /// Initializes a new instance of the <see cref="AvroAzureBatchJobSubmissionParameters"/> class. /// </summary> public AvroAzureBatchJobSubmissionParameters() http://git-wip-us.apache.org/repos/asf/reef/blob/ec738ad3/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/JobJarMaker.cs ---------------------------------------------------------------------- diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/JobJarMaker.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/JobJarMaker.cs index 48e62e5..4d434d7 100644 --- a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/JobJarMaker.cs +++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/JobJarMaker.cs @@ -40,11 +40,9 @@ namespace Org.Apache.REEF.Client.AzureBatch.Util IResourceArchiveFileGenerator resourceArchiveFileGenerator, DriverFolderPreparationHelper driverFolderPreparationHelper, REEFFileNames fileNames, - [Parameter(typeof(AzureBatchAccountKey))] string azureBatchAccountKey, [Parameter(typeof(AzureBatchAccountName))] string azureBatchAccountName, [Parameter(typeof(AzureBatchAccountUri))] string azureBatchAccountUri, [Parameter(typeof(AzureBatchPoolId))] string azureBatchPoolId, - [Parameter(typeof(AzureStorageAccountKey))] string azureStorageAccountKey, [Parameter(typeof(AzureStorageAccountName))] string azureStorageAccountName, [Parameter(typeof(AzureStorageContainerName))] string azureStorageContainerName) { @@ -53,14 +51,11 @@ namespace Org.Apache.REEF.Client.AzureBatch.Util _fileNames = fileNames; _avroAzureBatchJobSubmissionParameters = new AvroAzureBatchJobSubmissionParameters { - AzureBatchAccountKey = azureBatchAccountKey, AzureBatchAccountName = azureBatchAccountName, AzureBatchAccountUri = azureBatchAccountUri, AzureBatchPoolId = azureBatchPoolId, - AzureStorageAccountKey = azureStorageAccountKey, AzureStorageAccountName = azureStorageAccountName, AzureStorageContainerName = azureStorageContainerName, - AzureBatchIsWindows = true }; } http://git-wip-us.apache.org/repos/asf/reef/blob/ec738ad3/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc b/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc index e2a30f3..d3bda9f 100644 --- a/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc +++ b/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc @@ -73,14 +73,11 @@ "doc": "Cross-language submission parameters to the Azure Batch runtime", "fields": [ { "name": "sharedJobSubmissionParameters", "type": "AvroJobSubmissionParameters" }, - { "name": "AzureBatchAccountKey", "type": "string" }, { "name": "AzureBatchAccountName", "type": "string" }, { "name": "AzureBatchAccountUri", "type": "string" }, { "name": "AzureBatchPoolId", "type": "string" }, - { "name": "AzureStorageAccountKey", "type": "string" }, { "name": "AzureStorageAccountName", "type": "string" }, - { "name": "AzureStorageContainerName", "type": "string" }, - { "name": "AzureBatchIsWindows", "type": "boolean" } + { "name": "AzureStorageContainerName", "type": "string" } ] } ] http://git-wip-us.apache.org/repos/asf/reef/blob/ec738ad3/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java index c054722..5f09abd 100644 --- a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java +++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java @@ -18,9 +18,6 @@ */ package org.apache.reef.bridge.client; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.JsonDecoder; -import org.apache.avro.specific.SpecificDatumReader; import org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters; import org.apache.reef.runtime.common.client.DriverConfigurationProvider; import org.apache.reef.runtime.common.driver.parameters.ClientRemoteIdentifier; @@ -28,9 +25,6 @@ import org.apache.reef.tang.Configuration; import javax.inject.Inject; import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; import java.util.logging.Level; import java.util.logging.Logger; @@ -49,15 +43,10 @@ final class AzureBatchBootstrapDriverConfigGenerator { this.driverConfigurationProvider = driverConfigurationProvider; } - Configuration getDriverConfigurationFromParams(final String bootstrapJobArgsLocation) throws IOException { - - final File bootstrapJobArgsFile = new File(bootstrapJobArgsLocation).getCanonicalFile(); - - final AvroAzureBatchJobSubmissionParameters azureBatchBootstrapJobArgs = - readAzureBatchJobSubmissionParametersFromFile(bootstrapJobArgsFile); - - final String jobId = azureBatchBootstrapJobArgs.getSharedJobSubmissionParameters().getJobId().toString(); - final File jobFolder = new File(azureBatchBootstrapJobArgs + Configuration getDriverConfigurationFromParams( + final AvroAzureBatchJobSubmissionParameters avroAzureBatchJobSubmissionParameters) { + final String jobId = avroAzureBatchJobSubmissionParameters.getSharedJobSubmissionParameters().getJobId().toString(); + final File jobFolder = new File(avroAzureBatchJobSubmissionParameters .getSharedJobSubmissionParameters().getJobSubmissionFolder().toString()); LOG.log(Level.INFO, "jobFolder {0} jobId {1}.", new Object[]{jobFolder.toURI(), jobId}); @@ -66,20 +55,4 @@ final class AzureBatchBootstrapDriverConfigGenerator { jobFolder.toURI(), ClientRemoteIdentifier.NONE, jobId, Constants.DRIVER_CONFIGURATION_WITH_HTTP_AND_NAMESERVER); } - - private AvroAzureBatchJobSubmissionParameters readAzureBatchJobSubmissionParametersFromFile(final File file) - throws IOException { - try (final FileInputStream fileInputStream = new FileInputStream(file)) { - return readAzureBatchSubmissionParametersFromInputStream(fileInputStream); - } - } - - private static AvroAzureBatchJobSubmissionParameters readAzureBatchSubmissionParametersFromInputStream( - final InputStream inputStream) throws IOException { - final JsonDecoder decoder = DecoderFactory.get().jsonDecoder( - AvroAzureBatchJobSubmissionParameters.getClassSchema(), inputStream); - final SpecificDatumReader<AvroAzureBatchJobSubmissionParameters> reader = new SpecificDatumReader<>( - AvroAzureBatchJobSubmissionParameters.class); - return reader.read(null, decoder); - } } http://git-wip-us.apache.org/repos/asf/reef/blob/ec738ad3/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java ---------------------------------------------------------------------- diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java index 35a1e59..4f2a3f6 100644 --- a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java +++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java @@ -23,10 +23,21 @@ import org.apache.avro.io.JsonDecoder; import org.apache.avro.specific.SpecificDatumReader; import org.apache.reef.annotations.audience.Interop; import org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters; -import org.apache.reef.runtime.azbatch.client.AzureBatchRuntimeConfiguration; -import org.apache.reef.runtime.azbatch.client.AzureBatchRuntimeConfigurationCreator; +import org.apache.reef.runtime.azbatch.AzureBatchClasspathProvider; +import org.apache.reef.runtime.azbatch.AzureBatchJVMPathProvider; +import org.apache.reef.runtime.azbatch.client.AzureBatchDriverConfigurationProviderImpl; +import org.apache.reef.runtime.azbatch.parameters.AzureBatchAccountName; +import org.apache.reef.runtime.azbatch.parameters.AzureBatchAccountUri; +import org.apache.reef.runtime.azbatch.parameters.AzureBatchPoolId; +import org.apache.reef.runtime.azbatch.parameters.AzureStorageAccountName; +import org.apache.reef.runtime.azbatch.parameters.AzureStorageContainerName; +import org.apache.reef.runtime.azbatch.util.command.CommandBuilder; +import org.apache.reef.runtime.azbatch.util.command.WindowsCommandBuilder; import org.apache.reef.runtime.common.REEFEnvironment; +import org.apache.reef.runtime.common.client.DriverConfigurationProvider; import org.apache.reef.runtime.common.evaluator.PIDStoreStartHandler; +import org.apache.reef.runtime.common.files.RuntimeClasspathProvider; +import org.apache.reef.runtime.common.files.RuntimePathProvider; import org.apache.reef.runtime.common.launch.REEFErrorHandler; import org.apache.reef.runtime.common.launch.REEFMessageCodec; import org.apache.reef.tang.Configuration; @@ -73,9 +84,10 @@ public final class AzureBatchBootstrapREEFLauncher { throw fatal(message, new IllegalArgumentException(message)); } - final File partialConfigFile = new File(args[0]); + final AvroAzureBatchJobSubmissionParameters avroAzureBatchJobSubmissionParameters = + readAvroJobSubmissionParameters(new File(args[0])); final AzureBatchBootstrapDriverConfigGenerator azureBatchBootstrapDriverConfigGenerator = - TANG.newInjector(generateConfigurationFromJobSubmissionParameters(partialConfigFile)) + TANG.newInjector(generateConfiguration(avroAzureBatchJobSubmissionParameters)) .getInstance(AzureBatchBootstrapDriverConfigGenerator.class); final Configuration launcherConfig = @@ -87,7 +99,8 @@ public final class AzureBatchBootstrapREEFLauncher { .build(); try (final REEFEnvironment reef = REEFEnvironment.fromConfiguration( - azureBatchBootstrapDriverConfigGenerator.getDriverConfigurationFromParams(args[0]), launcherConfig)) { + azureBatchBootstrapDriverConfigGenerator.getDriverConfigurationFromParams( + avroAzureBatchJobSubmissionParameters), launcherConfig)) { reef.run(); } catch (final InjectionException ex) { throw fatal("Unable to configure and start REEFEnvironment.", ex); @@ -98,33 +111,35 @@ public final class AzureBatchBootstrapREEFLauncher { System.exit(0); // TODO[REEF-1715]: Should be able to exit cleanly at the end of main() } - private static Configuration generateConfigurationFromJobSubmissionParameters(final File params) throws IOException { - + private static AvroAzureBatchJobSubmissionParameters readAvroJobSubmissionParameters( + final File paramsFile) throws IOException { final AvroAzureBatchJobSubmissionParameters avroAzureBatchJobSubmissionParameters; - - try (final FileInputStream fileInputStream = new FileInputStream(params)) { + try (final FileInputStream fileInputStream = new FileInputStream(paramsFile)) { final JsonDecoder decoder = DecoderFactory.get().jsonDecoder( AvroAzureBatchJobSubmissionParameters.getClassSchema(), fileInputStream); final SpecificDatumReader<AvroAzureBatchJobSubmissionParameters> reader = new SpecificDatumReader<>(AvroAzureBatchJobSubmissionParameters.class); avroAzureBatchJobSubmissionParameters = reader.read(null, decoder); } + return avroAzureBatchJobSubmissionParameters; + } - return AzureBatchRuntimeConfigurationCreator - .getOrCreateAzureBatchRuntimeConfiguration(avroAzureBatchJobSubmissionParameters.getAzureBatchIsWindows()) - .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_ACCOUNT_NAME, + private static Configuration generateConfiguration( + final AvroAzureBatchJobSubmissionParameters avroAzureBatchJobSubmissionParameters) { + return TANG.newConfigurationBuilder() + .bindImplementation(DriverConfigurationProvider.class, AzureBatchDriverConfigurationProviderImpl.class) + .bindImplementation(RuntimeClasspathProvider.class, AzureBatchClasspathProvider.class) + .bindImplementation(RuntimePathProvider.class, AzureBatchJVMPathProvider.class) + .bindImplementation(CommandBuilder.class, WindowsCommandBuilder.class) + .bindNamedParameter(AzureBatchAccountName.class, avroAzureBatchJobSubmissionParameters.getAzureBatchAccountName().toString()) - .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_ACCOUNT_KEY, - avroAzureBatchJobSubmissionParameters.getAzureBatchAccountKey().toString()) - .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_ACCOUNT_URI, + .bindNamedParameter(AzureBatchAccountUri.class, avroAzureBatchJobSubmissionParameters.getAzureBatchAccountUri().toString()) - .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_POOL_ID, + .bindNamedParameter(AzureBatchPoolId.class, avroAzureBatchJobSubmissionParameters.getAzureBatchPoolId().toString()) - .set(AzureBatchRuntimeConfiguration.AZURE_STORAGE_ACCOUNT_NAME, + .bindNamedParameter(AzureStorageAccountName.class, avroAzureBatchJobSubmissionParameters.getAzureStorageAccountName().toString()) - .set(AzureBatchRuntimeConfiguration.AZURE_STORAGE_ACCOUNT_KEY, - avroAzureBatchJobSubmissionParameters.getAzureStorageAccountKey().toString()) - .set(AzureBatchRuntimeConfiguration.AZURE_STORAGE_CONTAINER_NAME, + .bindNamedParameter(AzureStorageContainerName.class, avroAzureBatchJobSubmissionParameters.getAzureStorageContainerName().toString()) .build(); }