[
https://issues.apache.org/jira/browse/BEAM-4814?focusedWorklogId=125216&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125216
]
ASF GitHub Bot logged work on BEAM-4814:
----------------------------------------
Author: ASF GitHub Bot
Created on: 19/Jul/18 19:56
Start Date: 19/Jul/18 19:56
Worklog Time Spent: 10m
Work Description: iemejia closed pull request #5983: [BEAM-4814] Add
client configuration to aws options
URL: https://github.com/apache/beam/pull/5983
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsModule.java
b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsModule.java
index 48bb74342c1..f78e08968eb 100644
---
a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsModule.java
+++
b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsModule.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.io.aws.options;
+import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
@@ -29,6 +30,9 @@
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
import com.amazonaws.services.s3.model.SSECustomerKey;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
@@ -68,6 +72,7 @@ public AwsModule() {
setMixInAnnotation(AWSCredentialsProvider.class,
AWSCredentialsProviderMixin.class);
setMixInAnnotation(SSECustomerKey.class, SSECustomerKeyMixin.class);
setMixInAnnotation(SSEAwsKeyManagementParams.class,
SSEAwsKeyManagementParamsMixin.class);
+ setMixInAnnotation(ClientConfiguration.class,
ClientConfigurationMixin.class);
}
/** A mixin to add Jackson annotations to {@link AWSCredentialsProvider}. */
@@ -234,4 +239,23 @@ public SSEAwsKeyManagementParams deserialize(JsonParser
parser, DeserializationC
return new SSEAwsKeyManagementParams(awsKmsKeyId);
}
}
+
+ @JsonAutoDetect(
+ fieldVisibility = Visibility.NONE,
+ getterVisibility = Visibility.NONE,
+ setterVisibility = Visibility.NONE
+ )
+ interface ClientConfigurationMixin {
+ @JsonProperty
+ String getProxyHost();
+
+ @JsonProperty
+ Integer getProxyPort();
+
+ @JsonProperty
+ String getProxyUsername();
+
+ @JsonProperty
+ String getProxyPassword();
+ }
}
diff --git
a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsOptions.java
b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsOptions.java
index 4f8969a11b4..65a6545f878 100644
---
a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsOptions.java
+++
b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsOptions.java
@@ -18,6 +18,7 @@
package org.apache.beam.sdk.io.aws.options;
+import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import org.apache.beam.sdk.options.Default;
@@ -75,4 +76,40 @@ public AWSCredentialsProvider create(PipelineOptions
options) {
return DefaultAWSCredentialsProviderChain.getInstance();
}
}
+
+ /**
+ * The client configuration instance that should be used to configure AWS
service clients. Please
+ * note that the configuration deserialization only allows one to specify
proxy settings.
+ *
+ * <p>For example, to specify the proxy host, port, username and password,
specify the following:
+ * <code>
+ * --clientConfiguration={
+ * "proxyHost":"hostname",
+ * "proxyPort":1234,
+ * "proxyUsername":"username",
+ * "proxyPassword":"password"
+ * }
+ * </code>
+ *
+ * @return
+ */
+ @Description(
+ "The client configuration instance that should be used to configure AWS
service "
+ + "clients. Please note that the configuration deserialization only
allows one to specify "
+ + "proxy settings. For example, to specify the proxy host, port,
username and password, "
+ + "specify the following:
--clientConfiguration={\"proxyHost\":\"hostname\",\"proxyPort\":1234,"
+ + "\"proxyUsername\":\"username\",\"proxyPassword\":\"password\"}")
+ @Default.InstanceFactory(ClientConfigurationFactory.class)
+ ClientConfiguration getClientConfiguration();
+
+ void setClientConfiguration(ClientConfiguration clientConfiguration);
+
+ /** Default AWS client configuration. */
+ class ClientConfigurationFactory implements
DefaultValueFactory<ClientConfiguration> {
+
+ @Override
+ public ClientConfiguration create(PipelineOptions options) {
+ return new ClientConfiguration();
+ }
+ }
}
diff --git
a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java
b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java
index 5b8c4d22da3..3332d37c80e 100644
---
a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java
+++
b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java
@@ -118,6 +118,11 @@
private static AmazonS3 buildAmazonS3Client(S3Options options) {
AmazonS3ClientBuilder builder =
AmazonS3ClientBuilder.standard().withCredentials(options.getAwsCredentialsProvider());
+
+ if (options.getClientConfiguration() != null) {
+ builder =
builder.withClientConfiguration(options.getClientConfiguration());
+ }
+
if (Strings.isNullOrEmpty(options.getAwsServiceEndpoint())) {
builder = builder.withRegion(options.getAwsRegion());
} else {
diff --git
a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/options/AwsModuleTest.java
b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/options/AwsModuleTest.java
index c9cf2b7fb12..04055f693a0 100644
---
a/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/options/AwsModuleTest.java
+++
b/sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/options/AwsModuleTest.java
@@ -22,6 +22,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
+import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
@@ -182,4 +183,21 @@ public void
testSSEAwsKeyManagementParamsSerializationDeserialization() throws E
assertEquals(awsKmsKeyId, valueDes.getAwsKmsKeyId());
assertEquals(encryption, valueDes.getEncryption());
}
+
+ @Test
+ public void testClientConfigurationSerializationDeserialization() throws
Exception {
+ ClientConfiguration clientConfiguration = new ClientConfiguration();
+ clientConfiguration.setProxyHost("localhost");
+ clientConfiguration.setProxyPort(1234);
+ clientConfiguration.setProxyUsername("username");
+ clientConfiguration.setProxyPassword("password");
+
+ final String valueAsJson =
objectMapper.writeValueAsString(clientConfiguration);
+ final ClientConfiguration valueDes =
+ objectMapper.readValue(valueAsJson, ClientConfiguration.class);
+ assertEquals("localhost", valueDes.getProxyHost());
+ assertEquals(1234, valueDes.getProxyPort());
+ assertEquals("username", valueDes.getProxyUsername());
+ assertEquals("password", valueDes.getProxyPassword());
+ }
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 125216)
Time Spent: 1.5h (was: 1h 20m)
> Support for S3FileSystem to work behind a proxy server
> ------------------------------------------------------
>
> Key: BEAM-4814
> URL: https://issues.apache.org/jira/browse/BEAM-4814
> Project: Beam
> Issue Type: Improvement
> Components: io-java-aws
> Reporter: John Rudolf Lewis
> Assignee: John Rudolf Lewis
> Priority: Major
> Time Spent: 1.5h
> Remaining Estimate: 0h
>
> I want to run a Beam job in my Spark cluster that uses the S3FileSystem. My
> Spark cluster is configured to require a proxy server with authentication in
> order to make outbound connections. A small change is required to enable
> ClientConfiguration to be added to the configuration to enable this.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
