This is an automated email from the ASF dual-hosted git repository.
maytasm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 161f4db Add integration tests for S3 Assume Role ingestion feature
(#11472)
161f4db is described below
commit 161f4dbc0ef5d429d8f16b43e0baf8efbacf5d13
Author: Maytas Monsereenusorn <[email protected]>
AuthorDate: Fri Jul 23 10:09:09 2021 +0700
Add integration tests for S3 Assume Role ingestion feature (#11472)
* add IT for S3 assume role
* fix checkstyle
* fix test
* fix pom
* fix test
---
integration-tests/pom.xml | 6 +
.../druid/testing/ConfigFileConfigProvider.java | 25 ++++
.../apache/druid/testing/DockerConfigProvider.java | 27 ++++
.../druid/testing/IntegrationTestingConfig.java | 6 +
...est.java => AbstractS3AssumeRoleIndexTest.java} | 143 +++++++++------------
.../tests/indexer/ITS3AssumeRoleIndexTest.java | 76 +++++++++++
...AssumeRoleWithOverrideCredentialsIndexTest.java | 77 +++++++++++
.../indexer/ITS3OverrideCredentialsIndexTest.java | 32 ++---
.../wikipedia_override_credentials_index_task.json | 5 +-
9 files changed, 290 insertions(+), 107 deletions(-)
diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml
index 188e9e4..df5becd 100644
--- a/integration-tests/pom.xml
+++ b/integration-tests/pom.xml
@@ -409,6 +409,12 @@
<version>0.9.3</version>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-sts</artifactId>
+ <version>${aws.sdk.version}</version>
+ <scope>runtime</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/integration-tests/src/main/java/org/apache/druid/testing/ConfigFileConfigProvider.java
b/integration-tests/src/main/java/org/apache/druid/testing/ConfigFileConfigProvider.java
index 7cd0387..e39c630 100644
---
a/integration-tests/src/main/java/org/apache/druid/testing/ConfigFileConfigProvider.java
+++
b/integration-tests/src/main/java/org/apache/druid/testing/ConfigFileConfigProvider.java
@@ -70,6 +70,9 @@ public class ConfigFileConfigProvider implements
IntegrationTestingConfigProvide
private String cloudBucket;
private String cloudPath;
private String cloudRegion;
+ private String s3AssumeRoleWithExternalId;
+ private String s3AssumeRoleExternalId;
+ private String s3AssumeRoleWithoutExternalId;
private String hadoopGcsCredentialsPath;
private String azureKey;
private String streamEndpoint;
@@ -232,6 +235,10 @@ public class ConfigFileConfigProvider implements
IntegrationTestingConfigProvide
cloudBucket = props.get("cloud_bucket");
cloudPath = props.get("cloud_path");
cloudRegion = props.get("cloud_region");
+ s3AssumeRoleWithExternalId = props.get("s3_assume_role_with_external_id");
+ s3AssumeRoleExternalId = props.get("s3_assume_role_external_id");
+ s3AssumeRoleWithoutExternalId =
props.get("s3_assume_role_without_external_id");
+
hadoopGcsCredentialsPath = props.get("hadoopGcsCredentialsPath");
azureKey = props.get("azureKey");
streamEndpoint = props.get("stream_endpoint");
@@ -484,6 +491,24 @@ public class ConfigFileConfigProvider implements
IntegrationTestingConfigProvide
}
@Override
+ public String getS3AssumeRoleWithExternalId()
+ {
+ return s3AssumeRoleWithExternalId;
+ }
+
+ @Override
+ public String getS3AssumeRoleExternalId()
+ {
+ return s3AssumeRoleExternalId;
+ }
+
+ @Override
+ public String getS3AssumeRoleWithoutExternalId()
+ {
+ return s3AssumeRoleWithoutExternalId;
+ }
+
+ @Override
public String getAzureKey()
{
return azureKey;
diff --git
a/integration-tests/src/main/java/org/apache/druid/testing/DockerConfigProvider.java
b/integration-tests/src/main/java/org/apache/druid/testing/DockerConfigProvider.java
index bb742a9..0e3a55d 100644
---
a/integration-tests/src/main/java/org/apache/druid/testing/DockerConfigProvider.java
+++
b/integration-tests/src/main/java/org/apache/druid/testing/DockerConfigProvider.java
@@ -58,6 +58,15 @@ public class DockerConfigProvider implements
IntegrationTestingConfigProvider
private String cloudRegion;
@JsonProperty
+ private String s3AssumeRoleWithExternalId;
+
+ @JsonProperty
+ private String s3AssumeRoleExternalId;
+
+ @JsonProperty
+ private String s3AssumeRoleWithoutExternalId;
+
+ @JsonProperty
private String hadoopGcsCredentialsPath;
@JsonProperty
@@ -391,6 +400,24 @@ public class DockerConfigProvider implements
IntegrationTestingConfigProvider
}
@Override
+ public String getS3AssumeRoleWithExternalId()
+ {
+ return s3AssumeRoleWithExternalId;
+ }
+
+ @Override
+ public String getS3AssumeRoleExternalId()
+ {
+ return s3AssumeRoleExternalId;
+ }
+
+ @Override
+ public String getS3AssumeRoleWithoutExternalId()
+ {
+ return s3AssumeRoleWithoutExternalId;
+ }
+
+ @Override
public String getAzureKey()
{
return azureKey;
diff --git
a/integration-tests/src/main/java/org/apache/druid/testing/IntegrationTestingConfig.java
b/integration-tests/src/main/java/org/apache/druid/testing/IntegrationTestingConfig.java
index b65507e..25d9580 100644
---
a/integration-tests/src/main/java/org/apache/druid/testing/IntegrationTestingConfig.java
+++
b/integration-tests/src/main/java/org/apache/druid/testing/IntegrationTestingConfig.java
@@ -158,6 +158,12 @@ public interface IntegrationTestingConfig
String getCloudRegion();
+ String getS3AssumeRoleWithExternalId();
+
+ String getS3AssumeRoleExternalId();
+
+ String getS3AssumeRoleWithoutExternalId();
+
String getAzureKey();
String getHadoopGcsCredentialsPath();
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3OverrideCredentialsIndexTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractS3AssumeRoleIndexTest.java
similarity index 63%
copy from
integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3OverrideCredentialsIndexTest.java
copy to
integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractS3AssumeRoleIndexTest.java
index 7991221..734845d 100644
---
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3OverrideCredentialsIndexTest.java
+++
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractS3AssumeRoleIndexTest.java
@@ -25,35 +25,16 @@ import org.apache.druid.indexer.TaskState;
import org.apache.druid.indexer.TaskStatusPlus;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.testing.guice.DruidTestModuleFactory;
-import org.apache.druid.tests.TestNGGroup;
import org.testng.Assert;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
+import org.testng.SkipException;
import java.io.Closeable;
import java.util.UUID;
import java.util.function.Function;
-/**
- * IMPORTANT:
- * To run this test, you must:
- * 1) Set the bucket and path for your data. This can be done by setting
-Ddruid.test.config.cloudBucket and
- * -Ddruid.test.config.cloudPath or setting "cloud_bucket" and "cloud_path"
in the config file.
- * 2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and
wikipedia_index_data3.json
- * located in integration-tests/src/test/resources/data/batch_index/json to
your S3 at the location set in step 1.
- * 3) Provide -Doverride.config.path=<PATH_TO_FILE> with s3
credentials/configs set. See
- * integration-tests/docker/environment-configs/override-examples/s3 for
env vars to provide.
- * Note that druid_s3_accessKey and druid_s3_secretKey should be unset or
set to credentials that does not have
- * access to the bucket and path specified in #1. The credentials that does
have access to the bucket and path
- * specified in #1 should be set to the env variable OVERRIDE_S3_ACCESS_KEY
and OVERRIDE_S3_SECRET_KEY
- */
-@Test(groups = TestNGGroup.S3_INGESTION)
-@Guice(moduleFactory = DruidTestModuleFactory.class)
-public class ITS3OverrideCredentialsIndexTest extends AbstractITBatchIndexTest
+public abstract class AbstractS3AssumeRoleIndexTest extends
AbstractITBatchIndexTest
{
private static final String INDEX_TASK_WITH_OVERRIDE =
"/indexer/wikipedia_override_credentials_index_task.json";
- private static final String INDEX_TASK_WITHOUT_OVERRIDE =
"/indexer/wikipedia_cloud_simple_index_task.json";
private static final String INDEX_QUERIES_RESOURCE =
"/indexer/wikipedia_index_queries.json";
private static final String INPUT_SOURCE_OBJECTS_KEY = "objects";
private static final String WIKIPEDIA_DATA_1 = "wikipedia_index_data1.json";
@@ -66,9 +47,13 @@ public class ITS3OverrideCredentialsIndexTest extends
AbstractITBatchIndexTest
ImmutableMap.of("bucket", "%%BUCKET%%", "path", "%%PATH%%" +
WIKIPEDIA_DATA_3)
);
- @Test
- public void testS3WithValidOverrideCredentialsIndexDataShouldSucceed()
throws Exception
+ abstract boolean isSetS3OverrideCredentials();
+
+ void doTestS3WithValidAssumeRoleAndExternalIdShouldSucceed() throws Exception
{
+ if (config.getS3AssumeRoleExternalId() == null ||
config.getS3AssumeRoleWithExternalId() == null) {
+ throw new SkipException("S3 Assume Role and external Id must be set for
this test");
+ }
final String indexDatasource = "wikipedia_index_test_" + UUID.randomUUID();
try (
final Closeable ignored1 = unloader(indexDatasource +
config.getExtraDatasourceNameSuffix());
@@ -86,24 +71,20 @@ public class ITS3OverrideCredentialsIndexTest extends
AbstractITBatchIndexTest
"%%PATH%%",
config.getCloudPath()
);
-
+ ImmutableMap.Builder<String, Object> s3ConfigMap =
ImmutableMap.builder();
+ if (isSetS3OverrideCredentials()) {
+ s3ConfigMap.put("accessKeyId", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_ACCESS_KEY"));
+ s3ConfigMap.put("secretAccessKey", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_SECRET_KEY"));
+ }
+ s3ConfigMap.put("assumeRoleArn",
config.getS3AssumeRoleWithExternalId());
+ s3ConfigMap.put("assumeRoleExternalId",
config.getS3AssumeRoleExternalId());
spec = StringUtils.replace(
spec,
- "%%ACCESS_KEY_PROPERTY_VALUE%%",
- jsonMapper.writeValueAsString(
- ImmutableMap.of("type", "environment", "variable",
"OVERRIDE_S3_ACCESS_KEY")
- )
+ "%%INPUT_SOURCE_CONFIG%%",
+ jsonMapper.writeValueAsString(s3ConfigMap.build())
);
spec = StringUtils.replace(
spec,
- "%%SECRET_KEY_PROPERTY_VALUE%%",
- jsonMapper.writeValueAsString(
- ImmutableMap.of("type", "environment", "variable",
"OVERRIDE_S3_SECRET_KEY")
- )
- );
-
- spec = StringUtils.replace(
- spec,
"%%INPUT_SOURCE_TYPE%%",
"s3"
);
@@ -136,9 +117,11 @@ public class ITS3OverrideCredentialsIndexTest extends
AbstractITBatchIndexTest
}
}
- @Test
- public void testS3WithoutOverrideCredentialsIndexDataShouldFailed() throws
Exception
+ void doTestS3WithAssumeRoleAndInvalidExternalIdShouldFail() throws Exception
{
+ if (config.getS3AssumeRoleExternalId() == null ||
config.getS3AssumeRoleWithExternalId() == null) {
+ throw new SkipException("S3 Assume Role and external Id must be set for
this test");
+ }
final String indexDatasource = "wikipedia_index_test_" + UUID.randomUUID();
try {
final Function<String, String> s3PropsTransform = spec -> {
@@ -154,7 +137,18 @@ public class ITS3OverrideCredentialsIndexTest extends
AbstractITBatchIndexTest
"%%PATH%%",
config.getCloudPath()
);
-
+ ImmutableMap.Builder<String, Object> s3ConfigMap =
ImmutableMap.builder();
+ if (isSetS3OverrideCredentials()) {
+ s3ConfigMap.put("accessKeyId", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_ACCESS_KEY"));
+ s3ConfigMap.put("secretAccessKey", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_SECRET_KEY"));
+ }
+ s3ConfigMap.put("assumeRoleArn",
config.getS3AssumeRoleWithExternalId());
+ s3ConfigMap.put("assumeRoleExternalId", "RANDOM_INVALID_VALUE_" +
UUID.randomUUID());
+ spec = StringUtils.replace(
+ spec,
+ "%%INPUT_SOURCE_CONFIG%%",
+ jsonMapper.writeValueAsString(s3ConfigMap.build())
+ );
spec = StringUtils.replace(
spec,
"%%INPUT_SOURCE_TYPE%%",
@@ -175,10 +169,11 @@ public class ITS3OverrideCredentialsIndexTest extends
AbstractITBatchIndexTest
throw new RuntimeException(e);
}
};
+
final String fullDatasourceName = indexDatasource +
config.getExtraDatasourceNameSuffix();
final String taskSpec = s3PropsTransform.apply(
StringUtils.replace(
- getResourceAsString(INDEX_TASK_WITHOUT_OVERRIDE),
+ getResourceAsString(INDEX_TASK_WITH_OVERRIDE),
"%%DATASOURCE%%",
fullDatasourceName
)
@@ -186,14 +181,12 @@ public class ITS3OverrideCredentialsIndexTest extends
AbstractITBatchIndexTest
final String taskID = indexer.submitTask(taskSpec);
indexer.waitUntilTaskFails(taskID);
TaskStatusPlus taskStatusPlus = indexer.getTaskStatus(taskID);
- // Index task is expected to fail as the default S3 Credentials in
Druid's config (druid.s3.accessKey and
- // druid.s3.secretKey should not have access to the bucket and path for
our data. (Refer to the setup instruction
- // at the top of this test class.
+ // Index task is expected to fail as the external id is invalid
Assert.assertEquals(taskStatusPlus.getStatusCode(), TaskState.FAILED);
Assert.assertNotNull(taskStatusPlus.getErrorMsg());
Assert.assertTrue(
-
taskStatusPlus.getErrorMsg().contains("com.amazonaws.services.s3.model.AmazonS3Exception"),
- "Expect task to fail with AmazonS3Exception");
+
taskStatusPlus.getErrorMsg().contains("com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException"),
+ "Expect task to fail with AWSSecurityTokenServiceException");
}
finally {
// If the test pass, then there is no datasource to unload
@@ -201,11 +194,15 @@ public class ITS3OverrideCredentialsIndexTest extends
AbstractITBatchIndexTest
}
}
- @Test
- public void testS3WithInvalidOverrideCredentialsIndexDataShouldFailed()
throws Exception
+ void doTestS3WithValidAssumeRoleWithoutExternalIdShouldSucceed() throws
Exception
{
+ if (config.getS3AssumeRoleWithoutExternalId() == null) {
+ throw new SkipException("S3 Assume Role must be set for this test");
+ }
final String indexDatasource = "wikipedia_index_test_" + UUID.randomUUID();
- try {
+ try (
+ final Closeable ignored1 = unloader(indexDatasource +
config.getExtraDatasourceNameSuffix());
+ ) {
final Function<String, String> s3PropsTransform = spec -> {
try {
String inputSourceValue =
jsonMapper.writeValueAsString(INPUT_SOURCE_OBJECTS_VALUE);
@@ -219,22 +216,17 @@ public class ITS3OverrideCredentialsIndexTest extends
AbstractITBatchIndexTest
"%%PATH%%",
config.getCloudPath()
);
-
- spec = StringUtils.replace(
- spec,
- "%%ACCESS_KEY_PROPERTY_VALUE%%",
- jsonMapper.writeValueAsString(
- ImmutableMap.of("type", "environment", "variable",
"NON_EXISTENT_INVALID_ENV_VAR")
- )
- );
+ ImmutableMap.Builder<String, Object> s3ConfigMap =
ImmutableMap.builder();
+ if (isSetS3OverrideCredentials()) {
+ s3ConfigMap.put("accessKeyId", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_ACCESS_KEY"));
+ s3ConfigMap.put("secretAccessKey", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_SECRET_KEY"));
+ }
+ s3ConfigMap.put("assumeRoleArn",
config.getS3AssumeRoleWithoutExternalId());
spec = StringUtils.replace(
spec,
- "%%SECRET_KEY_PROPERTY_VALUE%%",
- jsonMapper.writeValueAsString(
- ImmutableMap.of("type", "environment", "variable",
"NON_EXISTENT_INVALID_ENV_VAR")
- )
+ "%%INPUT_SOURCE_CONFIG%%",
+ jsonMapper.writeValueAsString(s3ConfigMap.build())
);
-
spec = StringUtils.replace(
spec,
"%%INPUT_SOURCE_TYPE%%",
@@ -256,27 +248,16 @@ public class ITS3OverrideCredentialsIndexTest extends
AbstractITBatchIndexTest
}
};
- final String fullDatasourceName = indexDatasource +
config.getExtraDatasourceNameSuffix();
- final String taskSpec = s3PropsTransform.apply(
- StringUtils.replace(
- getResourceAsString(INDEX_TASK_WITH_OVERRIDE),
- "%%DATASOURCE%%",
- fullDatasourceName
- )
+ doIndexTest(
+ indexDatasource,
+ INDEX_TASK_WITH_OVERRIDE,
+ s3PropsTransform,
+ INDEX_QUERIES_RESOURCE,
+ false,
+ true,
+ true,
+ new Pair<>(false, false)
);
- final String taskID = indexer.submitTask(taskSpec);
- indexer.waitUntilTaskFails(taskID);
- TaskStatusPlus taskStatusPlus = indexer.getTaskStatus(taskID);
- // Index task is expected to fail as the overrided s3 access key and s3
secret key cannot be null
- Assert.assertEquals(taskStatusPlus.getStatusCode(), TaskState.FAILED);
- Assert.assertNotNull(taskStatusPlus.getErrorMsg());
- Assert.assertTrue(
- taskStatusPlus.getErrorMsg().contains("IllegalArgumentException:
Access key cannot be null"),
- "Expect task to fail with IllegalArgumentException: Access key
cannot be null");
- }
- finally {
- // If the test pass, then there is no datasource to unload
- closeQuietly(unloader(indexDatasource +
config.getExtraDatasourceNameSuffix()));
}
}
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3AssumeRoleIndexTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3AssumeRoleIndexTest.java
new file mode 100644
index 0000000..ceede68
--- /dev/null
+++
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3AssumeRoleIndexTest.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.tests.indexer;
+
+import org.apache.druid.testing.guice.DruidTestModuleFactory;
+import org.apache.druid.tests.TestNGGroup;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+/**
+ * IMPORTANT:
+ * To run this test, you must:
+ * 1) Set the bucket and path for your data. This can be done by setting
-Ddruid.test.config.cloudBucket and
+ * -Ddruid.test.config.cloudPath or setting "cloud_bucket" and "cloud_path"
in the config file.
+ * 2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and
wikipedia_index_data3.json
+ * located in integration-tests/src/test/resources/data/batch_index/json to
your S3 at the location set in step 1.
+ * 3) Provide -Doverride.config.path=<PATH_TO_FILE> with s3
credentials/configs set. See
+ * integration-tests/docker/environment-configs/override-examples/s3 for
env vars to provide.
+ * Note that druid_s3_accessKey and druid_s3_secretKey should be set to
credentials that does have
+ * access to the role in #4.
+ * 4) Set the assume role configs. This can be done by setting
+ * -Ddruid.test.config.s3AssumeRoleWithExternalId or setting
"s3_assume_role_with_external_id" in the config file.
+ * -Ddruid.test.config.s3AssumeRoleExternalId or setting
"s3_assume_role_external_id" in the config file.
+ * -Ddruid.test.config.s3AssumeRoleWithoutExternalId or setting
"s3_assume_role_without_external_id" in the config file.
+ * The credientials provided in #3 must be able to assume these roles.
+ * These roles must also have access to the bucket and path for your data
in #1.
+ * (s3AssumeRoleExternalId is the external id for
s3AssumeRoleWithExternalId, while s3AssumeRoleWithoutExternalId
+ * should not have external id set)
+ *
+ * NOTE: Tests in this class will be skipped if properties in #4 are not
set.
+ */
+@Test(groups = TestNGGroup.S3_DEEP_STORAGE)
+@Guice(moduleFactory = DruidTestModuleFactory.class)
+public class ITS3AssumeRoleIndexTest extends AbstractS3AssumeRoleIndexTest
+{
+ @Override
+ public boolean isSetS3OverrideCredentials()
+ {
+ return false;
+ }
+
+ @Test
+ public void testS3WithValidAssumeRoleAndExternalIdShouldSucceed() throws
Exception
+ {
+ doTestS3WithValidAssumeRoleAndExternalIdShouldSucceed();
+ }
+
+ @Test
+ public void testS3WithAssumeRoleAndInvalidExternalIdShouldFail() throws
Exception
+ {
+ doTestS3WithAssumeRoleAndInvalidExternalIdShouldFail();
+ }
+
+ @Test
+ public void testS3WithValidAssumeRoleWithoutExternalIdShouldSucceed() throws
Exception
+ {
+ doTestS3WithValidAssumeRoleWithoutExternalIdShouldSucceed();
+ }
+}
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3AssumeRoleWithOverrideCredentialsIndexTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3AssumeRoleWithOverrideCredentialsIndexTest.java
new file mode 100644
index 0000000..6650e2f
--- /dev/null
+++
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3AssumeRoleWithOverrideCredentialsIndexTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.tests.indexer;
+
+import org.apache.druid.testing.guice.DruidTestModuleFactory;
+import org.apache.druid.tests.TestNGGroup;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+/**
+ * IMPORTANT:
+ * To run this test, you must:
+ * 1) Set the bucket and path for your data. This can be done by setting
-Ddruid.test.config.cloudBucket and
+ * -Ddruid.test.config.cloudPath or setting "cloud_bucket" and "cloud_path"
in the config file.
+ * 2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and
wikipedia_index_data3.json
+ * located in integration-tests/src/test/resources/data/batch_index/json to
your S3 at the location set in step 1.
+ * 3) Provide -Doverride.config.path=<PATH_TO_FILE> with s3
credentials/configs set. See
+ * integration-tests/docker/environment-configs/override-examples/s3 for
env vars to provide.
+ * Note that druid_s3_accessKey and druid_s3_secretKey should be unset or
set to credentials that does not have
+ * access to the role. The credentials that does have access to the role
should be set to the env variable
+ * OVERRIDE_S3_ACCESS_KEY and OVERRIDE_S3_SECRET_KEY
+ * 4) Set the assume role configs. This can be done by setting
+ * -Ddruid.test.config.s3AssumeRoleWithExternalId or setting
"s3_assume_role_with_external_id" in the config file.
+ * -Ddruid.test.config.s3AssumeRoleExternalId or setting
"s3_assume_role_external_id" in the config file.
+ * -Ddruid.test.config.s3AssumeRoleWithoutExternalId or setting
"s3_assume_role_without_external_id" in the config file.
+ * The credientials provided in OVERRIDE_S3_ACCESS_KEY and
OVERRIDE_S3_SECRET_KEY must be able to assume these roles.
+ * These roles must also have access to the bucket and path for your data
in #1.
+ * (s3AssumeRoleExternalId is the external id for
s3AssumeRoleWithExternalId, while s3AssumeRoleWithoutExternalId
+ * should not have external id set)
+ *
+ * NOTE: Tests in this class will be skipped if properties in #4 are not
set.
+ */
+@Test(groups = TestNGGroup.S3_INGESTION)
+@Guice(moduleFactory = DruidTestModuleFactory.class)
+public class ITS3AssumeRoleWithOverrideCredentialsIndexTest extends
AbstractS3AssumeRoleIndexTest
+{
+ @Override
+ public boolean isSetS3OverrideCredentials()
+ {
+ return true;
+ }
+
+ @Test
+ public void
testS3WithValidAssumeRoleAndExternalIdUsingOverrideCredentialsShouldSucceed()
throws Exception
+ {
+ doTestS3WithValidAssumeRoleAndExternalIdShouldSucceed();
+ }
+
+ @Test
+ public void
testS3WithAssumeRoleAndInvalidExternalIdUsingOverrideCredentialsShouldFail()
throws Exception
+ {
+ doTestS3WithAssumeRoleAndInvalidExternalIdShouldFail();
+ }
+
+ @Test
+ public void
testS3WithValidAssumeRoleWithoutExternalIdUsingOverrideCredentialsShouldSucceed()
throws Exception
+ {
+ doTestS3WithValidAssumeRoleWithoutExternalIdShouldSucceed();
+ }
+}
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3OverrideCredentialsIndexTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3OverrideCredentialsIndexTest.java
index 7991221..f2561d6 100644
---
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3OverrideCredentialsIndexTest.java
+++
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3OverrideCredentialsIndexTest.java
@@ -86,24 +86,18 @@ public class ITS3OverrideCredentialsIndexTest extends
AbstractITBatchIndexTest
"%%PATH%%",
config.getCloudPath()
);
-
spec = StringUtils.replace(
spec,
- "%%ACCESS_KEY_PROPERTY_VALUE%%",
+ "%%INPUT_SOURCE_CONFIG%%",
jsonMapper.writeValueAsString(
- ImmutableMap.of("type", "environment", "variable",
"OVERRIDE_S3_ACCESS_KEY")
+ ImmutableMap.of(
+ "accessKeyId", ImmutableMap.of("type", "environment",
"variable", "OVERRIDE_S3_ACCESS_KEY"),
+ "secretAccessKey", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_SECRET_KEY")
+ )
)
);
spec = StringUtils.replace(
spec,
- "%%SECRET_KEY_PROPERTY_VALUE%%",
- jsonMapper.writeValueAsString(
- ImmutableMap.of("type", "environment", "variable",
"OVERRIDE_S3_SECRET_KEY")
- )
- );
-
- spec = StringUtils.replace(
- spec,
"%%INPUT_SOURCE_TYPE%%",
"s3"
);
@@ -219,24 +213,18 @@ public class ITS3OverrideCredentialsIndexTest extends
AbstractITBatchIndexTest
"%%PATH%%",
config.getCloudPath()
);
-
spec = StringUtils.replace(
spec,
- "%%ACCESS_KEY_PROPERTY_VALUE%%",
+ "%%INPUT_SOURCE_CONFIG%%",
jsonMapper.writeValueAsString(
- ImmutableMap.of("type", "environment", "variable",
"NON_EXISTENT_INVALID_ENV_VAR")
+ ImmutableMap.of(
+ "accessKeyId", ImmutableMap.of("type", "environment",
"variable", "INVALID_ACCESS_KEY"),
+ "secretAccessKey", ImmutableMap.of("type",
"environment", "variable", "INVALID_SECRET_KEY")
+ )
)
);
spec = StringUtils.replace(
spec,
- "%%SECRET_KEY_PROPERTY_VALUE%%",
- jsonMapper.writeValueAsString(
- ImmutableMap.of("type", "environment", "variable",
"NON_EXISTENT_INVALID_ENV_VAR")
- )
- );
-
- spec = StringUtils.replace(
- spec,
"%%INPUT_SOURCE_TYPE%%",
"s3"
);
diff --git
a/integration-tests/src/test/resources/indexer/wikipedia_override_credentials_index_task.json
b/integration-tests/src/test/resources/indexer/wikipedia_override_credentials_index_task.json
index 3db2dcc..12b8797 100644
---
a/integration-tests/src/test/resources/indexer/wikipedia_override_credentials_index_task.json
+++
b/integration-tests/src/test/resources/indexer/wikipedia_override_credentials_index_task.json
@@ -68,10 +68,7 @@
"type": "index",
"inputSource": {
"type": "%%INPUT_SOURCE_TYPE%%",
- "properties": {
- "accessKeyId": %%ACCESS_KEY_PROPERTY_VALUE%%,
- "secretAccessKey": %%SECRET_KEY_PROPERTY_VALUE%%
- },
+ "properties": %%INPUT_SOURCE_CONFIG%%,
"%%INPUT_SOURCE_PROPERTY_KEY%%": %%INPUT_SOURCE_PROPERTY_VALUE%%
},
"inputFormat": {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
