[beam] branch master updated: [BEAM-3550] Add --awsServiceEndpoint option and use with S3 filesystem.

[lcwik]

This is an automated email from the ASF dual-hosted git repository.

lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new bc3bc68  [BEAM-3550] Add --awsServiceEndpoint option and use with S3 
filesystem.
bc3bc68 is described below

commit bc3bc68974e4e16fc4f30247695041e22a9d4a6a
Author: John MacMillan <john...@ca.ibm.com>
AuthorDate: Mon Jan 29 15:24:59 2018 -0500

    [BEAM-3550] Add --awsServiceEndpoint option and use with S3 filesystem.
    
    If the option is null or empty, the default endpoint will be used.
---
 .../org/apache/beam/sdk/io/aws/options/AwsOptions.java    |  7 +++++++
 .../java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java  | 15 ++++++++++-----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git 
a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsOptions.java
 
b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsOptions.java
index 8ed68e5..ce55792 100644
--- 
a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsOptions.java
+++ 
b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsOptions.java
@@ -40,6 +40,13 @@ public interface AwsOptions extends PipelineOptions {
   void setAwsRegion(String value);
 
   /**
+   * The AWS service endpoint used by the AWS client.
+   */
+  @Description("AWS service endpoint used by the AWS client")
+  String getAwsServiceEndpoint();
+  void setAwsServiceEndpoint(String value);
+
+  /**
    * The credential instance that should be used to authenticate against AWS 
services. Refer to
    * {@link DefaultAWSCredentialsProviderChain} Javadoc for usage help.
    */
diff --git 
a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java
 
b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java
index 11c3418..5adf42a 100644
--- 
a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java
+++ 
b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java
@@ -22,6 +22,7 @@ import static 
com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 
 import com.amazonaws.AmazonClientException;
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.AmazonS3Exception;
@@ -106,11 +107,15 @@ class S3FileSystem extends FileSystem<S3ResourceId> {
               + "was not specified. If you don't plan to use S3, then ignore 
this message.");
     }
 
-    amazonS3 =
-        AmazonS3ClientBuilder.standard()
-            .withCredentials(options.getAwsCredentialsProvider())
-            .withRegion(options.getAwsRegion())
-            .build();
+    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard()
+        .withCredentials(options.getAwsCredentialsProvider());
+    if (Strings.isNullOrEmpty(options.getAwsServiceEndpoint())) {
+      builder = builder.withRegion(options.getAwsRegion());
+    } else {
+      builder = builder.withEndpointConfiguration(new EndpointConfiguration(
+          options.getAwsServiceEndpoint(), options.getAwsRegion()));
+    }
+    amazonS3 = builder.build();
 
     this.storageClass = checkNotNull(options.getS3StorageClass(), 
"storageClass");
 

-- 
To stop receiving notification emails like this one, please contact
lc...@apache.org.