This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 6c24c50e460 Move ITS3AssumeRoleIndexTest and
ITS3OverrideCredentialsIndexTest (#19108)
6c24c50e460 is described below
commit 6c24c50e460de8ceb81b0b576d2f439484706a9d
Author: Kashif Faraz <[email protected]>
AuthorDate: Sat Mar 7 17:34:29 2026 +0530
Move ITS3AssumeRoleIndexTest and ITS3OverrideCredentialsIndexTest (#19108)
Changes:
- Add method in `MinIOStorageResource` to create temporary credentials
using `AssumeRoleRequest`
- Add test method to `ITS3ToS3ParallelIndexTest` to test ingestion with
temporary override credentials
- It is currently not supported to test
`assumeRoleArn`/`assumeRoleExternalId` using Min IO
- Remove old integration tests
---
embedded-tests/pom.xml | 5 +
.../azure/ITAzureToAzureParallelIndexTest.java | 2 +-
.../embedded/azure/ITAzureV2ParallelIndexTest.java | 2 +-
.../embedded/gcs/ITGcsToGcsParallelIndexTest.java | 2 +-
.../AbstractCloudInputSourceParallelIndexTest.java | 10 +-
.../AbstractS3InputSourceParallelIndexTest.java | 2 +-
.../embedded/minio/ITS3ToS3ParallelIndexTest.java | 11 +-
.../embedded/minio/MinIOStorageResource.java | 51 ++++
.../indexer/wikipedia_cloud_index_task.json | 1 +
.../java/org/apache/druid/tests/TestNGGroup.java | 7 -
.../indexer/AbstractS3AssumeRoleIndexTest.java | 274 --------------------
.../tests/indexer/ITS3AssumeRoleIndexTest.java | 76 ------
...AssumeRoleWithOverrideCredentialsIndexTest.java | 77 ------
.../indexer/ITS3OverrideCredentialsIndexTest.java | 281 ---------------------
.../wikipedia_override_credentials_index_task.json | 82 ------
15 files changed, 80 insertions(+), 803 deletions(-)
diff --git a/embedded-tests/pom.xml b/embedded-tests/pom.xml
index aa04be53017..192bfa71b03 100644
--- a/embedded-tests/pom.xml
+++ b/embedded-tests/pom.xml
@@ -540,6 +540,11 @@
<artifactId>aws-java-sdk-core</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-sts</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-kinesis</artifactId>
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/azure/ITAzureToAzureParallelIndexTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/azure/ITAzureToAzureParallelIndexTest.java
index 126c43670fd..fa056811324 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/azure/ITAzureToAzureParallelIndexTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/azure/ITAzureToAzureParallelIndexTest.java
@@ -31,6 +31,6 @@ public class ITAzureToAzureParallelIndexTest extends
AbstractAzureInputSourcePar
@MethodSource("resources")
public void testAzureIndexData(Pair<String, List<?>> azureInputSource)
throws Exception
{
- doTest(azureInputSource, new Pair<>(false, false), "azure");
+ doTest(azureInputSource, new Pair<>(false, false), "azure", null);
}
}
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/azure/ITAzureV2ParallelIndexTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/azure/ITAzureV2ParallelIndexTest.java
index eeba91f4155..afbf6878e94 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/azure/ITAzureV2ParallelIndexTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/azure/ITAzureV2ParallelIndexTest.java
@@ -31,7 +31,7 @@ public class ITAzureV2ParallelIndexTest extends
AbstractAzureInputSourceParallel
@MethodSource("resources")
public void testAzureIndexData(Pair<String, List<?>> azureInputSource)
throws Exception
{
- String dataSource = doTest(azureInputSource, new Pair<>(false, false),
"azureStorage");
+ String dataSource = doTest(azureInputSource, new Pair<>(false, false),
"azureStorage", null);
validateAzureSegmentFilesDeleted("segments" + "/" + dataSource);
}
}
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/gcs/ITGcsToGcsParallelIndexTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/gcs/ITGcsToGcsParallelIndexTest.java
index 53494a2d2b8..06bc10d743f 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/gcs/ITGcsToGcsParallelIndexTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/gcs/ITGcsToGcsParallelIndexTest.java
@@ -31,6 +31,6 @@ public class ITGcsToGcsParallelIndexTest extends
AbstractGcsInputSourceParallelI
@MethodSource("resources")
public void testGcsIndexData(Pair<String, List<?>> gcsInputSource) throws
Exception
{
- doTest(gcsInputSource, new Pair<>(false, false), "google");
+ doTest(gcsInputSource, new Pair<>(false, false), "google", null);
}
}
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/AbstractCloudInputSourceParallelIndexTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/AbstractCloudInputSourceParallelIndexTest.java
index c409de50b91..7498ca4a5f1 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/AbstractCloudInputSourceParallelIndexTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/AbstractCloudInputSourceParallelIndexTest.java
@@ -126,7 +126,8 @@ public abstract class
AbstractCloudInputSourceParallelIndexTest extends Abstract
protected String doTest(
Pair<String, List<?>> inputSource,
Pair<Boolean, Boolean> segmentAvailabilityConfirmationPair,
- String inputSourceType
+ String inputSourceType,
+ Object inputSourceProperties
) throws Exception
{
final String indexDatasource = dataSource;
@@ -162,6 +163,13 @@ public abstract class
AbstractCloudInputSourceParallelIndexTest extends Abstract
"%%INPUT_SOURCE_TYPE%%",
inputSourceType
);
+
+ final String properties =
jsonMapper.writeValueAsString(inputSourceProperties);
+ spec = StringUtils.replace(
+ spec,
+ "%%INPUT_SOURCE_PROPERTIES%%",
+ properties
+ );
spec = StringUtils.replace(
spec,
"%%INPUT_SOURCE_PROPERTY_KEY%%",
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/AbstractS3InputSourceParallelIndexTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/AbstractS3InputSourceParallelIndexTest.java
index 384e1d8bc26..7a4e47ad768 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/AbstractS3InputSourceParallelIndexTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/AbstractS3InputSourceParallelIndexTest.java
@@ -41,7 +41,7 @@ import java.util.List;
public abstract class AbstractS3InputSourceParallelIndexTest extends
AbstractCloudInputSourceParallelIndexTest
{
private static final Logger LOG = new
Logger(AbstractS3InputSourceParallelIndexTest.class);
- private final MinIOStorageResource minIOStorageResource = new
MinIOStorageResource();
+ protected final MinIOStorageResource minIOStorageResource = new
MinIOStorageResource();
private S3TestUtil s3;
@Override
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/ITS3ToS3ParallelIndexTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/ITS3ToS3ParallelIndexTest.java
index fd22260ecc3..98584cd9b2e 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/ITS3ToS3ParallelIndexTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/ITS3ToS3ParallelIndexTest.java
@@ -19,6 +19,7 @@
package org.apache.druid.testing.embedded.minio;
+import org.apache.druid.data.input.s3.S3InputSourceConfig;
import org.apache.druid.java.util.common.Pair;
import
org.apache.druid.testing.embedded.indexer.AbstractS3InputSourceParallelIndexTest;
import org.junit.jupiter.params.ParameterizedTest;
@@ -32,6 +33,14 @@ public class ITS3ToS3ParallelIndexTest extends
AbstractS3InputSourceParallelInde
@MethodSource("resources")
public void testS3IndexData(Pair<String, List<?>> s3InputSource) throws
Exception
{
- doTest(s3InputSource, new Pair<>(false, false), "s3");
+ doTest(s3InputSource, new Pair<>(false, false), "s3", null);
+ }
+
+ @ParameterizedTest
+ @MethodSource("resources")
+ public void testS3IndexData_withTempCredentials(Pair<String, List<?>>
s3InputSource) throws Exception
+ {
+ final S3InputSourceConfig inputSourceConfig =
minIOStorageResource.createTempCredentialsForInputSource();
+ doTest(s3InputSource, new Pair<>(false, false), "s3", inputSourceConfig);
}
}
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResource.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResource.java
index d16a61319d8..7eece0c7a4d 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResource.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResource.java
@@ -24,7 +24,15 @@ import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
+import
com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
+import com.amazonaws.services.securitytoken.model.AssumeRoleRequest;
+import com.amazonaws.services.securitytoken.model.AssumeRoleResult;
+import com.amazonaws.services.securitytoken.model.Credentials;
import org.apache.druid.common.aws.AWSModule;
+import org.apache.druid.data.input.s3.S3InputSourceConfig;
+import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.metadata.DefaultPasswordProvider;
import org.apache.druid.storage.s3.S3StorageDruidModule;
import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
import org.apache.druid.testing.embedded.TestcontainerResource;
@@ -124,6 +132,40 @@ public class MinIOStorageResource extends
TestcontainerResource<MinIOContainer>
return s3Client;
}
+ /**
+ * Creates temporary S3 credentials using the {@link AssumeRoleRequest} that
+ * can be used for S3 ingestion.
+ *
+ * @return S3InputSourceConfig with temporary credentials. The
+ * {@code assumeRoleArn} and {@code assumeRoleExternalId} fields are set to
null
+ * since Min IO does not support them.
+ */
+ public S3InputSourceConfig createTempCredentialsForInputSource()
+ {
+ ensureRunning();
+
+ final AWSSecurityTokenService stsClient = createSTSClient();
+
+ // assumeRoleArn and assumeRoleExternalId need not be specified since
MinIO ignores them
+ final AssumeRoleRequest assumeRoleRequest = new AssumeRoleRequest()
+ .withRoleSessionName("test-session");
+
+ final AssumeRoleResult result = stsClient.assumeRole(assumeRoleRequest);
+ final int statusCode = result.getSdkHttpMetadata().getHttpStatusCode();
+ if (statusCode < 200 || statusCode >= 300) {
+ throw new ISE("AssumeRole request failed with code[%s]: %s", statusCode,
result.getAssumedRoleUser());
+ }
+
+ final Credentials credentials =
stsClient.assumeRole(assumeRoleRequest).getCredentials();
+ return new S3InputSourceConfig(
+ new DefaultPasswordProvider(credentials.getAccessKeyId()),
+ new DefaultPasswordProvider(credentials.getSecretAccessKey()),
+ null,
+ null,
+ new DefaultPasswordProvider(credentials.getSessionToken())
+ );
+ }
+
private AmazonS3 createS3Client()
{
return AmazonS3Client
@@ -133,4 +175,13 @@ public class MinIOStorageResource extends
TestcontainerResource<MinIOContainer>
.withPathStyleAccessEnabled(true)
.build();
}
+
+ private AWSSecurityTokenService createSTSClient()
+ {
+ return AWSSecurityTokenServiceClientBuilder
+ .standard()
+ .withEndpointConfiguration(new
AwsClientBuilder.EndpointConfiguration(getEndpointUrl(), "us-east-1"))
+ .withCredentials(new AWSStaticCredentialsProvider(new
BasicAWSCredentials(getAccessKey(), getSecretKey())))
+ .build();
+ }
}
diff --git
a/embedded-tests/src/test/resources/indexer/wikipedia_cloud_index_task.json
b/embedded-tests/src/test/resources/indexer/wikipedia_cloud_index_task.json
index 8c4242c91e2..671c7d358e2 100644
--- a/embedded-tests/src/test/resources/indexer/wikipedia_cloud_index_task.json
+++ b/embedded-tests/src/test/resources/indexer/wikipedia_cloud_index_task.json
@@ -68,6 +68,7 @@
"type": "index_parallel",
"inputSource": {
"type": "%%INPUT_SOURCE_TYPE%%",
+ "properties": %%INPUT_SOURCE_PROPERTIES%%,
"%%INPUT_SOURCE_PROPERTY_KEY%%": %%INPUT_SOURCE_PROPERTY_VALUE%%
},
"inputFormat": {
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java
b/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java
index 23bd0ebe189..178e55cb2e9 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java
@@ -72,11 +72,4 @@ public class TestNGGroup
public static final String HADOOP_GCS_TO_GCS =
"hadoop-gcs-to-gcs-deep-storage";
public static final String HADOOP_GCS_TO_HDFS =
"hadoop-gcs-to-hdfs-deep-storage";
-
- /**
- * This group is not part of CI. To run this group, s3 configs/credentials
for your s3 must be provided in a file.
- * The path of the file must then be pass to mvn with
-Doverride.config.path=<PATH_TO_FILE>
- * See integration-tests/docker/environment-configs/override-examples/s3 for
env vars to provide.
- */
- public static final String S3_INGESTION = "s3-ingestion";
}
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractS3AssumeRoleIndexTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractS3AssumeRoleIndexTest.java
deleted file mode 100644
index 734845dea53..00000000000
---
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractS3AssumeRoleIndexTest.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.tests.indexer;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.apache.druid.indexer.TaskState;
-import org.apache.druid.indexer.TaskStatusPlus;
-import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.StringUtils;
-import org.testng.Assert;
-import org.testng.SkipException;
-
-import java.io.Closeable;
-import java.util.UUID;
-import java.util.function.Function;
-
-public abstract class AbstractS3AssumeRoleIndexTest extends
AbstractITBatchIndexTest
-{
- private static final String INDEX_TASK_WITH_OVERRIDE =
"/indexer/wikipedia_override_credentials_index_task.json";
- private static final String INDEX_QUERIES_RESOURCE =
"/indexer/wikipedia_index_queries.json";
- private static final String INPUT_SOURCE_OBJECTS_KEY = "objects";
- private static final String WIKIPEDIA_DATA_1 = "wikipedia_index_data1.json";
- private static final String WIKIPEDIA_DATA_2 = "wikipedia_index_data2.json";
- private static final String WIKIPEDIA_DATA_3 = "wikipedia_index_data3.json";
- private static final ImmutableList INPUT_SOURCE_OBJECTS_VALUE =
ImmutableList.of
- (
- ImmutableMap.of("bucket", "%%BUCKET%%", "path", "%%PATH%%" +
WIKIPEDIA_DATA_1),
- ImmutableMap.of("bucket", "%%BUCKET%%", "path", "%%PATH%%" +
WIKIPEDIA_DATA_2),
- ImmutableMap.of("bucket", "%%BUCKET%%", "path", "%%PATH%%" +
WIKIPEDIA_DATA_3)
- );
-
- abstract boolean isSetS3OverrideCredentials();
-
- void doTestS3WithValidAssumeRoleAndExternalIdShouldSucceed() throws Exception
- {
- if (config.getS3AssumeRoleExternalId() == null ||
config.getS3AssumeRoleWithExternalId() == null) {
- throw new SkipException("S3 Assume Role and external Id must be set for
this test");
- }
- final String indexDatasource = "wikipedia_index_test_" + UUID.randomUUID();
- try (
- final Closeable ignored1 = unloader(indexDatasource +
config.getExtraDatasourceNameSuffix());
- ) {
- final Function<String, String> s3PropsTransform = spec -> {
- try {
- String inputSourceValue =
jsonMapper.writeValueAsString(INPUT_SOURCE_OBJECTS_VALUE);
- inputSourceValue = StringUtils.replace(
- inputSourceValue,
- "%%BUCKET%%",
- config.getCloudBucket()
- );
- inputSourceValue = StringUtils.replace(
- inputSourceValue,
- "%%PATH%%",
- config.getCloudPath()
- );
- ImmutableMap.Builder<String, Object> s3ConfigMap =
ImmutableMap.builder();
- if (isSetS3OverrideCredentials()) {
- s3ConfigMap.put("accessKeyId", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_ACCESS_KEY"));
- s3ConfigMap.put("secretAccessKey", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_SECRET_KEY"));
- }
- s3ConfigMap.put("assumeRoleArn",
config.getS3AssumeRoleWithExternalId());
- s3ConfigMap.put("assumeRoleExternalId",
config.getS3AssumeRoleExternalId());
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_CONFIG%%",
- jsonMapper.writeValueAsString(s3ConfigMap.build())
- );
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_TYPE%%",
- "s3"
- );
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_PROPERTY_KEY%%",
- INPUT_SOURCE_OBJECTS_KEY
- );
- return StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_PROPERTY_VALUE%%",
- inputSourceValue
- );
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
-
- doIndexTest(
- indexDatasource,
- INDEX_TASK_WITH_OVERRIDE,
- s3PropsTransform,
- INDEX_QUERIES_RESOURCE,
- false,
- true,
- true,
- new Pair<>(false, false)
- );
- }
- }
-
- void doTestS3WithAssumeRoleAndInvalidExternalIdShouldFail() throws Exception
- {
- if (config.getS3AssumeRoleExternalId() == null ||
config.getS3AssumeRoleWithExternalId() == null) {
- throw new SkipException("S3 Assume Role and external Id must be set for
this test");
- }
- final String indexDatasource = "wikipedia_index_test_" + UUID.randomUUID();
- try {
- final Function<String, String> s3PropsTransform = spec -> {
- try {
- String inputSourceValue =
jsonMapper.writeValueAsString(INPUT_SOURCE_OBJECTS_VALUE);
- inputSourceValue = StringUtils.replace(
- inputSourceValue,
- "%%BUCKET%%",
- config.getCloudBucket()
- );
- inputSourceValue = StringUtils.replace(
- inputSourceValue,
- "%%PATH%%",
- config.getCloudPath()
- );
- ImmutableMap.Builder<String, Object> s3ConfigMap =
ImmutableMap.builder();
- if (isSetS3OverrideCredentials()) {
- s3ConfigMap.put("accessKeyId", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_ACCESS_KEY"));
- s3ConfigMap.put("secretAccessKey", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_SECRET_KEY"));
- }
- s3ConfigMap.put("assumeRoleArn",
config.getS3AssumeRoleWithExternalId());
- s3ConfigMap.put("assumeRoleExternalId", "RANDOM_INVALID_VALUE_" +
UUID.randomUUID());
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_CONFIG%%",
- jsonMapper.writeValueAsString(s3ConfigMap.build())
- );
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_TYPE%%",
- "s3"
- );
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_PROPERTY_KEY%%",
- INPUT_SOURCE_OBJECTS_KEY
- );
- return StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_PROPERTY_VALUE%%",
- inputSourceValue
- );
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
-
- final String fullDatasourceName = indexDatasource +
config.getExtraDatasourceNameSuffix();
- final String taskSpec = s3PropsTransform.apply(
- StringUtils.replace(
- getResourceAsString(INDEX_TASK_WITH_OVERRIDE),
- "%%DATASOURCE%%",
- fullDatasourceName
- )
- );
- final String taskID = indexer.submitTask(taskSpec);
- indexer.waitUntilTaskFails(taskID);
- TaskStatusPlus taskStatusPlus = indexer.getTaskStatus(taskID);
- // Index task is expected to fail as the external id is invalid
- Assert.assertEquals(taskStatusPlus.getStatusCode(), TaskState.FAILED);
- Assert.assertNotNull(taskStatusPlus.getErrorMsg());
- Assert.assertTrue(
-
taskStatusPlus.getErrorMsg().contains("com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException"),
- "Expect task to fail with AWSSecurityTokenServiceException");
- }
- finally {
- // If the test pass, then there is no datasource to unload
- closeQuietly(unloader(indexDatasource +
config.getExtraDatasourceNameSuffix()));
- }
- }
-
- void doTestS3WithValidAssumeRoleWithoutExternalIdShouldSucceed() throws
Exception
- {
- if (config.getS3AssumeRoleWithoutExternalId() == null) {
- throw new SkipException("S3 Assume Role must be set for this test");
- }
- final String indexDatasource = "wikipedia_index_test_" + UUID.randomUUID();
- try (
- final Closeable ignored1 = unloader(indexDatasource +
config.getExtraDatasourceNameSuffix());
- ) {
- final Function<String, String> s3PropsTransform = spec -> {
- try {
- String inputSourceValue =
jsonMapper.writeValueAsString(INPUT_SOURCE_OBJECTS_VALUE);
- inputSourceValue = StringUtils.replace(
- inputSourceValue,
- "%%BUCKET%%",
- config.getCloudBucket()
- );
- inputSourceValue = StringUtils.replace(
- inputSourceValue,
- "%%PATH%%",
- config.getCloudPath()
- );
- ImmutableMap.Builder<String, Object> s3ConfigMap =
ImmutableMap.builder();
- if (isSetS3OverrideCredentials()) {
- s3ConfigMap.put("accessKeyId", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_ACCESS_KEY"));
- s3ConfigMap.put("secretAccessKey", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_SECRET_KEY"));
- }
- s3ConfigMap.put("assumeRoleArn",
config.getS3AssumeRoleWithoutExternalId());
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_CONFIG%%",
- jsonMapper.writeValueAsString(s3ConfigMap.build())
- );
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_TYPE%%",
- "s3"
- );
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_PROPERTY_KEY%%",
- INPUT_SOURCE_OBJECTS_KEY
- );
- return StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_PROPERTY_VALUE%%",
- inputSourceValue
- );
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
-
- doIndexTest(
- indexDatasource,
- INDEX_TASK_WITH_OVERRIDE,
- s3PropsTransform,
- INDEX_QUERIES_RESOURCE,
- false,
- true,
- true,
- new Pair<>(false, false)
- );
- }
- }
-
- private void closeQuietly(Closeable closeable)
- {
- try {
- if (closeable != null) {
- closeable.close();
- }
- }
- catch (Exception var2) {
- }
- }
-}
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3AssumeRoleIndexTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3AssumeRoleIndexTest.java
deleted file mode 100644
index ceede68d080..00000000000
---
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3AssumeRoleIndexTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.tests.indexer;
-
-import org.apache.druid.testing.guice.DruidTestModuleFactory;
-import org.apache.druid.tests.TestNGGroup;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-/**
- * IMPORTANT:
- * To run this test, you must:
- * 1) Set the bucket and path for your data. This can be done by setting
-Ddruid.test.config.cloudBucket and
- * -Ddruid.test.config.cloudPath or setting "cloud_bucket" and "cloud_path"
in the config file.
- * 2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and
wikipedia_index_data3.json
- * located in integration-tests/src/test/resources/data/batch_index/json to
your S3 at the location set in step 1.
- * 3) Provide -Doverride.config.path=<PATH_TO_FILE> with s3
credentials/configs set. See
- * integration-tests/docker/environment-configs/override-examples/s3 for
env vars to provide.
- * Note that druid_s3_accessKey and druid_s3_secretKey should be set to
credentials that does have
- * access to the role in #4.
- * 4) Set the assume role configs. This can be done by setting
- * -Ddruid.test.config.s3AssumeRoleWithExternalId or setting
"s3_assume_role_with_external_id" in the config file.
- * -Ddruid.test.config.s3AssumeRoleExternalId or setting
"s3_assume_role_external_id" in the config file.
- * -Ddruid.test.config.s3AssumeRoleWithoutExternalId or setting
"s3_assume_role_without_external_id" in the config file.
- * The credientials provided in #3 must be able to assume these roles.
- * These roles must also have access to the bucket and path for your data
in #1.
- * (s3AssumeRoleExternalId is the external id for
s3AssumeRoleWithExternalId, while s3AssumeRoleWithoutExternalId
- * should not have external id set)
- *
- * NOTE: Tests in this class will be skipped if properties in #4 are not
set.
- */
-@Test(groups = TestNGGroup.S3_DEEP_STORAGE)
-@Guice(moduleFactory = DruidTestModuleFactory.class)
-public class ITS3AssumeRoleIndexTest extends AbstractS3AssumeRoleIndexTest
-{
- @Override
- public boolean isSetS3OverrideCredentials()
- {
- return false;
- }
-
- @Test
- public void testS3WithValidAssumeRoleAndExternalIdShouldSucceed() throws
Exception
- {
- doTestS3WithValidAssumeRoleAndExternalIdShouldSucceed();
- }
-
- @Test
- public void testS3WithAssumeRoleAndInvalidExternalIdShouldFail() throws
Exception
- {
- doTestS3WithAssumeRoleAndInvalidExternalIdShouldFail();
- }
-
- @Test
- public void testS3WithValidAssumeRoleWithoutExternalIdShouldSucceed() throws
Exception
- {
- doTestS3WithValidAssumeRoleWithoutExternalIdShouldSucceed();
- }
-}
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3AssumeRoleWithOverrideCredentialsIndexTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3AssumeRoleWithOverrideCredentialsIndexTest.java
deleted file mode 100644
index 6d4ac025d34..00000000000
---
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3AssumeRoleWithOverrideCredentialsIndexTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.tests.indexer;
-
-import org.apache.druid.testing.guice.DruidTestModuleFactory;
-import org.apache.druid.tests.TestNGGroup;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-/**
- * IMPORTANT:
- * To run this test, you must:
- * 1) Set the bucket and path for your data. This can be done by setting
-Ddruid.test.config.cloudBucket and
- * -Ddruid.test.config.cloudPath or setting "cloud_bucket" and "cloud_path"
in the config file.
- * 2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and
wikipedia_index_data3.json
- * located in integration-tests/src/test/resources/data/batch_index/json to
your S3 at the location set in step 1.
- * 3) Provide -Doverride.config.path=<PATH_TO_FILE> with s3
credentials/configs set. See
- * integration-tests/docker/environment-configs/override-examples/s3 for
env vars to provide.
- * Note that druid_s3_accessKey and druid_s3_secretKey should be unset or
set to credentials that does not have
- * access to the role. The credentials that does have access to the role
should be set to the env variable
- * OVERRIDE_S3_ACCESS_KEY and OVERRIDE_S3_SECRET_KEY
- * 4) Set the assume role configs. This can be done by setting
- * -Ddruid.test.config.s3AssumeRoleWithExternalId or setting
"s3_assume_role_with_external_id" in the config file.
- * -Ddruid.test.config.s3AssumeRoleExternalId or setting
"s3_assume_role_external_id" in the config file.
- * -Ddruid.test.config.s3AssumeRoleWithoutExternalId or setting
"s3_assume_role_without_external_id" in the config file.
- * The credentials provided in OVERRIDE_S3_ACCESS_KEY and
OVERRIDE_S3_SECRET_KEY must be able to assume these roles.
- * These roles must also have access to the bucket and path for your data
in #1.
- * (s3AssumeRoleExternalId is the external id for
s3AssumeRoleWithExternalId, while s3AssumeRoleWithoutExternalId
- * should not have external id set)
- *
- * NOTE: Tests in this class will be skipped if properties in #4 are not
set.
- */
-@Test(groups = TestNGGroup.S3_INGESTION)
-@Guice(moduleFactory = DruidTestModuleFactory.class)
-public class ITS3AssumeRoleWithOverrideCredentialsIndexTest extends
AbstractS3AssumeRoleIndexTest
-{
- @Override
- public boolean isSetS3OverrideCredentials()
- {
- return true;
- }
-
- @Test
- public void
testS3WithValidAssumeRoleAndExternalIdUsingOverrideCredentialsShouldSucceed()
throws Exception
- {
- doTestS3WithValidAssumeRoleAndExternalIdShouldSucceed();
- }
-
- @Test
- public void
testS3WithAssumeRoleAndInvalidExternalIdUsingOverrideCredentialsShouldFail()
throws Exception
- {
- doTestS3WithAssumeRoleAndInvalidExternalIdShouldFail();
- }
-
- @Test
- public void
testS3WithValidAssumeRoleWithoutExternalIdUsingOverrideCredentialsShouldSucceed()
throws Exception
- {
- doTestS3WithValidAssumeRoleWithoutExternalIdShouldSucceed();
- }
-}
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3OverrideCredentialsIndexTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3OverrideCredentialsIndexTest.java
deleted file mode 100644
index f2561d67d0d..00000000000
---
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITS3OverrideCredentialsIndexTest.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.tests.indexer;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.apache.druid.indexer.TaskState;
-import org.apache.druid.indexer.TaskStatusPlus;
-import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.testing.guice.DruidTestModuleFactory;
-import org.apache.druid.tests.TestNGGroup;
-import org.testng.Assert;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import java.io.Closeable;
-import java.util.UUID;
-import java.util.function.Function;
-
-/**
- * IMPORTANT:
- * To run this test, you must:
- * 1) Set the bucket and path for your data. This can be done by setting
-Ddruid.test.config.cloudBucket and
- * -Ddruid.test.config.cloudPath or setting "cloud_bucket" and "cloud_path"
in the config file.
- * 2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and
wikipedia_index_data3.json
- * located in integration-tests/src/test/resources/data/batch_index/json to
your S3 at the location set in step 1.
- * 3) Provide -Doverride.config.path=<PATH_TO_FILE> with s3
credentials/configs set. See
- * integration-tests/docker/environment-configs/override-examples/s3 for
env vars to provide.
- * Note that druid_s3_accessKey and druid_s3_secretKey should be unset or
set to credentials that does not have
- * access to the bucket and path specified in #1. The credentials that does
have access to the bucket and path
- * specified in #1 should be set to the env variable OVERRIDE_S3_ACCESS_KEY
and OVERRIDE_S3_SECRET_KEY
- */
-@Test(groups = TestNGGroup.S3_INGESTION)
-@Guice(moduleFactory = DruidTestModuleFactory.class)
-public class ITS3OverrideCredentialsIndexTest extends AbstractITBatchIndexTest
-{
- private static final String INDEX_TASK_WITH_OVERRIDE =
"/indexer/wikipedia_override_credentials_index_task.json";
- private static final String INDEX_TASK_WITHOUT_OVERRIDE =
"/indexer/wikipedia_cloud_simple_index_task.json";
- private static final String INDEX_QUERIES_RESOURCE =
"/indexer/wikipedia_index_queries.json";
- private static final String INPUT_SOURCE_OBJECTS_KEY = "objects";
- private static final String WIKIPEDIA_DATA_1 = "wikipedia_index_data1.json";
- private static final String WIKIPEDIA_DATA_2 = "wikipedia_index_data2.json";
- private static final String WIKIPEDIA_DATA_3 = "wikipedia_index_data3.json";
- private static final ImmutableList INPUT_SOURCE_OBJECTS_VALUE =
ImmutableList.of
- (
- ImmutableMap.of("bucket", "%%BUCKET%%", "path", "%%PATH%%" +
WIKIPEDIA_DATA_1),
- ImmutableMap.of("bucket", "%%BUCKET%%", "path", "%%PATH%%" +
WIKIPEDIA_DATA_2),
- ImmutableMap.of("bucket", "%%BUCKET%%", "path", "%%PATH%%" +
WIKIPEDIA_DATA_3)
- );
-
- @Test
- public void testS3WithValidOverrideCredentialsIndexDataShouldSucceed()
throws Exception
- {
- final String indexDatasource = "wikipedia_index_test_" + UUID.randomUUID();
- try (
- final Closeable ignored1 = unloader(indexDatasource +
config.getExtraDatasourceNameSuffix());
- ) {
- final Function<String, String> s3PropsTransform = spec -> {
- try {
- String inputSourceValue =
jsonMapper.writeValueAsString(INPUT_SOURCE_OBJECTS_VALUE);
- inputSourceValue = StringUtils.replace(
- inputSourceValue,
- "%%BUCKET%%",
- config.getCloudBucket()
- );
- inputSourceValue = StringUtils.replace(
- inputSourceValue,
- "%%PATH%%",
- config.getCloudPath()
- );
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_CONFIG%%",
- jsonMapper.writeValueAsString(
- ImmutableMap.of(
- "accessKeyId", ImmutableMap.of("type", "environment",
"variable", "OVERRIDE_S3_ACCESS_KEY"),
- "secretAccessKey", ImmutableMap.of("type",
"environment", "variable", "OVERRIDE_S3_SECRET_KEY")
- )
- )
- );
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_TYPE%%",
- "s3"
- );
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_PROPERTY_KEY%%",
- INPUT_SOURCE_OBJECTS_KEY
- );
- return StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_PROPERTY_VALUE%%",
- inputSourceValue
- );
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
-
- doIndexTest(
- indexDatasource,
- INDEX_TASK_WITH_OVERRIDE,
- s3PropsTransform,
- INDEX_QUERIES_RESOURCE,
- false,
- true,
- true,
- new Pair<>(false, false)
- );
- }
- }
-
- @Test
- public void testS3WithoutOverrideCredentialsIndexDataShouldFailed() throws
Exception
- {
- final String indexDatasource = "wikipedia_index_test_" + UUID.randomUUID();
- try {
- final Function<String, String> s3PropsTransform = spec -> {
- try {
- String inputSourceValue =
jsonMapper.writeValueAsString(INPUT_SOURCE_OBJECTS_VALUE);
- inputSourceValue = StringUtils.replace(
- inputSourceValue,
- "%%BUCKET%%",
- config.getCloudBucket()
- );
- inputSourceValue = StringUtils.replace(
- inputSourceValue,
- "%%PATH%%",
- config.getCloudPath()
- );
-
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_TYPE%%",
- "s3"
- );
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_PROPERTY_KEY%%",
- INPUT_SOURCE_OBJECTS_KEY
- );
- return StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_PROPERTY_VALUE%%",
- inputSourceValue
- );
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
- final String fullDatasourceName = indexDatasource +
config.getExtraDatasourceNameSuffix();
- final String taskSpec = s3PropsTransform.apply(
- StringUtils.replace(
- getResourceAsString(INDEX_TASK_WITHOUT_OVERRIDE),
- "%%DATASOURCE%%",
- fullDatasourceName
- )
- );
- final String taskID = indexer.submitTask(taskSpec);
- indexer.waitUntilTaskFails(taskID);
- TaskStatusPlus taskStatusPlus = indexer.getTaskStatus(taskID);
- // Index task is expected to fail as the default S3 Credentials in
Druid's config (druid.s3.accessKey and
- // druid.s3.secretKey should not have access to the bucket and path for
our data. (Refer to the setup instruction
- // at the top of this test class.
- Assert.assertEquals(taskStatusPlus.getStatusCode(), TaskState.FAILED);
- Assert.assertNotNull(taskStatusPlus.getErrorMsg());
- Assert.assertTrue(
-
taskStatusPlus.getErrorMsg().contains("com.amazonaws.services.s3.model.AmazonS3Exception"),
- "Expect task to fail with AmazonS3Exception");
- }
- finally {
- // If the test pass, then there is no datasource to unload
- closeQuietly(unloader(indexDatasource +
config.getExtraDatasourceNameSuffix()));
- }
- }
-
- @Test
- public void testS3WithInvalidOverrideCredentialsIndexDataShouldFailed()
throws Exception
- {
- final String indexDatasource = "wikipedia_index_test_" + UUID.randomUUID();
- try {
- final Function<String, String> s3PropsTransform = spec -> {
- try {
- String inputSourceValue =
jsonMapper.writeValueAsString(INPUT_SOURCE_OBJECTS_VALUE);
- inputSourceValue = StringUtils.replace(
- inputSourceValue,
- "%%BUCKET%%",
- config.getCloudBucket()
- );
- inputSourceValue = StringUtils.replace(
- inputSourceValue,
- "%%PATH%%",
- config.getCloudPath()
- );
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_CONFIG%%",
- jsonMapper.writeValueAsString(
- ImmutableMap.of(
- "accessKeyId", ImmutableMap.of("type", "environment",
"variable", "INVALID_ACCESS_KEY"),
- "secretAccessKey", ImmutableMap.of("type",
"environment", "variable", "INVALID_SECRET_KEY")
- )
- )
- );
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_TYPE%%",
- "s3"
- );
- spec = StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_PROPERTY_KEY%%",
- INPUT_SOURCE_OBJECTS_KEY
- );
- return StringUtils.replace(
- spec,
- "%%INPUT_SOURCE_PROPERTY_VALUE%%",
- inputSourceValue
- );
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
-
- final String fullDatasourceName = indexDatasource +
config.getExtraDatasourceNameSuffix();
- final String taskSpec = s3PropsTransform.apply(
- StringUtils.replace(
- getResourceAsString(INDEX_TASK_WITH_OVERRIDE),
- "%%DATASOURCE%%",
- fullDatasourceName
- )
- );
- final String taskID = indexer.submitTask(taskSpec);
- indexer.waitUntilTaskFails(taskID);
- TaskStatusPlus taskStatusPlus = indexer.getTaskStatus(taskID);
- // Index task is expected to fail as the overrided s3 access key and s3
secret key cannot be null
- Assert.assertEquals(taskStatusPlus.getStatusCode(), TaskState.FAILED);
- Assert.assertNotNull(taskStatusPlus.getErrorMsg());
- Assert.assertTrue(
- taskStatusPlus.getErrorMsg().contains("IllegalArgumentException:
Access key cannot be null"),
- "Expect task to fail with IllegalArgumentException: Access key
cannot be null");
- }
- finally {
- // If the test pass, then there is no datasource to unload
- closeQuietly(unloader(indexDatasource +
config.getExtraDatasourceNameSuffix()));
- }
- }
-
- private void closeQuietly(Closeable closeable)
- {
- try {
- if (closeable != null) {
- closeable.close();
- }
- }
- catch (Exception var2) {
- }
- }
-}
diff --git
a/integration-tests/src/test/resources/indexer/wikipedia_override_credentials_index_task.json
b/integration-tests/src/test/resources/indexer/wikipedia_override_credentials_index_task.json
deleted file mode 100644
index 12b87977743..00000000000
---
a/integration-tests/src/test/resources/indexer/wikipedia_override_credentials_index_task.json
+++ /dev/null
@@ -1,82 +0,0 @@
-{
- "type": "index",
- "spec": {
- "dataSchema": {
- "dataSource": "%%DATASOURCE%%",
- "timestampSpec": {
- "column": "timestamp"
- },
- "dimensionsSpec": {
- "dimensions": [
- "page",
- {"type": "string", "name": "language", "createBitmapIndex": false},
- "user",
- "unpatrolled",
- "newPage",
- "robot",
- "anonymous",
- "namespace",
- "continent",
- "country",
- "region",
- "city"
- ]
- },
- "metricsSpec": [
- {
- "type": "count",
- "name": "count"
- },
- {
- "type": "doubleSum",
- "name": "added",
- "fieldName": "added"
- },
- {
- "type": "doubleSum",
- "name": "deleted",
- "fieldName": "deleted"
- },
- {
- "type": "doubleSum",
- "name": "delta",
- "fieldName": "delta"
- },
- {
- "name": "thetaSketch",
- "type": "thetaSketch",
- "fieldName": "user"
- },
- {
- "name": "quantilesDoublesSketch",
- "type": "quantilesDoublesSketch",
- "fieldName": "delta"
- },
- {
- "name": "HLLSketchBuild",
- "type": "HLLSketchBuild",
- "fieldName": "user"
- }
- ],
- "granularitySpec": {
- "segmentGranularity": "DAY",
- "queryGranularity": "second",
- "intervals" : [ "2013-08-31/2013-09-02" ]
- }
- },
- "ioConfig": {
- "type": "index",
- "inputSource": {
- "type": "%%INPUT_SOURCE_TYPE%%",
- "properties": %%INPUT_SOURCE_CONFIG%%,
- "%%INPUT_SOURCE_PROPERTY_KEY%%": %%INPUT_SOURCE_PROPERTY_VALUE%%
- },
- "inputFormat": {
- "type": "json"
- }
- },
- "tuningConfig": {
- "type": "index"
- }
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
