paul-rogers commented on code in PR #13535:
URL: https://github.com/apache/druid/pull/13535#discussion_r1053774387
##########
integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/IntegrationTestingConfigEx.java:
##########
@@ -353,6 +354,18 @@ public String getCloudBucket()
return getProperty("cloudBucket");
}
+ @Override
+ public String getCloudBucketwhenDeepStorageTypeIs(String storageType)
+ {
+ if (Objects.equals(storageType, "s3")) {
+ return getProperty("cloudBucket");
+ } else if (Objects.equals(storageType, "azure")) {
Review Comment:
Might want to declare constants for these strings so that we can be sure
that the caller and this method use the same strings.
##########
integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractCloudInputSourceParallelIndexTest.java:
##########
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testsEx.indexer;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
+import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.logger.Logger;
+
+import java.io.Closeable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Function;
+
+/**
+ * Common abstract class for most cloud deep storage tests
+ */
+public abstract class AbstractCloudInputSourceParallelIndexTest extends
AbstractITBatchIndexTest
+{
+ private static final String INDEX_TASK =
"/indexer/wikipedia_cloud_index_task.json";
+ private static final String INDEX_QUERIES_RESOURCE =
"/indexer/wikipedia_index_queries.json";
+ private static final String INPUT_SOURCE_URIS_KEY = "uris";
+ private static final String INPUT_SOURCE_PREFIXES_KEY = "prefixes";
+ private static final String INPUT_SOURCE_OBJECTS_KEY = "objects";
+ private static final String WIKIPEDIA_DATA_1 = "wikipedia_index_data1.json";
+ private static final String WIKIPEDIA_DATA_2 = "wikipedia_index_data2.json";
+ private static final String WIKIPEDIA_DATA_3 = "wikipedia_index_data3.json";
+ private static final Logger LOG = new
Logger(AbstractCloudInputSourceParallelIndexTest.class);
+
+ String indexDatasource = "wikipedia_cloud_index_test_";
+
+ public static Object[][] resources()
+ {
+ return new Object[][]{
+ {new Pair<>(INPUT_SOURCE_URIS_KEY,
+ ImmutableList.of(
+ "%%INPUT_SOURCE_TYPE%%://%%BUCKET%%/%%PATH%%/" +
WIKIPEDIA_DATA_1,
+ "%%INPUT_SOURCE_TYPE%%://%%BUCKET%%/%%PATH%%/" +
WIKIPEDIA_DATA_2,
+ "%%INPUT_SOURCE_TYPE%%://%%BUCKET%%/%%PATH%%/" +
WIKIPEDIA_DATA_3
+ )
+ )},
+ {new Pair<>(INPUT_SOURCE_PREFIXES_KEY,
+ ImmutableList.of(
+ "%%INPUT_SOURCE_TYPE%%://%%BUCKET%%/%%PATH%%/"
+ )
+ )},
+ {new Pair<>(INPUT_SOURCE_OBJECTS_KEY,
+ ImmutableList.of(
+ ImmutableMap.of("bucket", "%%BUCKET%%", "path",
"%%PATH%%/" + WIKIPEDIA_DATA_1),
+ ImmutableMap.of("bucket", "%%BUCKET%%", "path",
"%%PATH%%/" + WIKIPEDIA_DATA_2),
+ ImmutableMap.of("bucket", "%%BUCKET%%", "path",
"%%PATH%%/" + WIKIPEDIA_DATA_3)
+ )
+ )}
+ };
+ }
+
+ public static List<String> fileList()
+ {
+ return Arrays.asList(WIKIPEDIA_DATA_1, WIKIPEDIA_DATA_2, WIKIPEDIA_DATA_3);
+ }
+
+ public String setInputSourceInPath(String inputSourceType, String
inputSourcePath)
+ {
+ if ("google".equals(inputSourceType)) {
Review Comment:
Please pull "google" out as a constant.
##########
integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractS3InputSourceParallelIndexTest.java:
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testsEx.indexer;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
+import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.testing.utils.S3TestUtil;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.Closeable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Function;
+
+import static junit.framework.Assert.fail;
+
+
+public abstract class AbstractS3InputSourceParallelIndexTest extends
AbstractITBatchIndexTest
+{
+ private static String indexDatasource;
+ private static S3TestUtil s3;
+ private static final String INDEX_TASK =
"/indexer/wikipedia_cloud_index_task.json";
+ private static final String INDEX_QUERIES_RESOURCE =
"/indexer/wikipedia_index_queries.json";
+ private static final String INPUT_SOURCE_URIS_KEY = "uris";
+ private static final String INPUT_SOURCE_PREFIXES_KEY = "prefixes";
+ private static final String INPUT_SOURCE_OBJECTS_KEY = "objects";
+ private static final String WIKIPEDIA_DATA_1 = "wikipedia_index_data1.json";
+ private static final String WIKIPEDIA_DATA_2 = "wikipedia_index_data2.json";
+ private static final String WIKIPEDIA_DATA_3 = "wikipedia_index_data3.json";
+
+ public static Object[][] resources()
+ {
+ return new Object[][]{
+ {new Pair<>(INPUT_SOURCE_URIS_KEY,
+ ImmutableList.of(
+ "s3://%%BUCKET%%/%%PATH%%/" + WIKIPEDIA_DATA_1,
+ "s3://%%BUCKET%%/%%PATH%%/" + WIKIPEDIA_DATA_2,
+ "s3://%%BUCKET%%/%%PATH%%/" + WIKIPEDIA_DATA_3
+ )
+ )},
+ {new Pair<>(INPUT_SOURCE_PREFIXES_KEY,
+ ImmutableList.of(
+ "s3://%%BUCKET%%/%%PATH%%/"
+ )
+ )},
+ {new Pair<>(INPUT_SOURCE_OBJECTS_KEY,
+ ImmutableList.of(
+ ImmutableMap.of("bucket", "%%BUCKET%%", "path",
"%%PATH%%/" + WIKIPEDIA_DATA_1),
+ ImmutableMap.of("bucket", "%%BUCKET%%", "path",
"%%PATH%%/" + WIKIPEDIA_DATA_2),
+ ImmutableMap.of("bucket", "%%BUCKET%%", "path",
"%%PATH%%/" + WIKIPEDIA_DATA_3)
+ )
+ )}
+ };
+ }
+
+ public static List<String> fileList()
+ {
+ return Arrays.asList(WIKIPEDIA_DATA_1, WIKIPEDIA_DATA_2, WIKIPEDIA_DATA_3);
+ }
+
+ @BeforeClass
+ public static void uploadDataFilesToS3()
+ {
+ List<String> filesToUpload = new ArrayList<>();
+ String localPath = "resources/data/batch_index/json/";
+ for (String file : fileList()) {
+ filesToUpload.add(localPath + file);
+ }
+ try {
+ s3 = new S3TestUtil();
+ s3.uploadDataFilesToS3(filesToUpload);
Review Comment:
I suppose this code won't run anywhere except MacOS/Linux, so it is probably
OK to hard-code assumptions about paths...
##########
integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractCloudInputSourceParallelIndexTest.java:
##########
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testsEx.indexer;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
+import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.logger.Logger;
+
+import java.io.Closeable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Function;
+
+/**
+ * Common abstract class for most cloud deep storage tests
+ */
+public abstract class AbstractCloudInputSourceParallelIndexTest extends
AbstractITBatchIndexTest
+{
+ private static final String INDEX_TASK =
"/indexer/wikipedia_cloud_index_task.json";
+ private static final String INDEX_QUERIES_RESOURCE =
"/indexer/wikipedia_index_queries.json";
+ private static final String INPUT_SOURCE_URIS_KEY = "uris";
+ private static final String INPUT_SOURCE_PREFIXES_KEY = "prefixes";
+ private static final String INPUT_SOURCE_OBJECTS_KEY = "objects";
+ private static final String WIKIPEDIA_DATA_1 = "wikipedia_index_data1.json";
+ private static final String WIKIPEDIA_DATA_2 = "wikipedia_index_data2.json";
+ private static final String WIKIPEDIA_DATA_3 = "wikipedia_index_data3.json";
+ private static final Logger LOG = new
Logger(AbstractCloudInputSourceParallelIndexTest.class);
+
+ String indexDatasource = "wikipedia_cloud_index_test_";
Review Comment:
Nit: this can be a constant: it doesn't vary per instance.
##########
integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/IntegrationTestingConfigEx.java:
##########
@@ -353,6 +354,18 @@ public String getCloudBucket()
return getProperty("cloudBucket");
}
+ @Override
+ public String getCloudBucketwhenDeepStorageTypeIs(String storageType)
+ {
+ if (Objects.equals(storageType, "s3")) {
Review Comment:
`"s3".equals(storageType)` is the typical Java pattern.
##########
integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractITBatchIndexTest.java:
##########
@@ -93,14 +99,149 @@ public String getFolderSuffix()
private static final Logger LOG = new Logger(AbstractITBatchIndexTest.class);
- @Inject
- protected IntegrationTestingConfig config;
@Inject
protected SqlTestQueryHelper sqlQueryHelper;
@Inject
ClientInfoResourceTestClient clientInfoResourceTestClient;
+ @Inject
+ private MsqTestQueryHelper msqHelper;
+
+ @Inject
+ protected TestQueryHelper queryHelper;
+
+ @Inject
+ private DataLoaderHelper dataLoaderHelper;
+
+ @Rule
+ public TestWatcher watchman = new TestWatcher()
+ {
+ @Override
+ public void starting(Description d)
+ {
+ LOG.info("RUNNING %s", d.getDisplayName());
+ }
+
+ @Override
+ public void failed(Throwable e, Description d)
+ {
+ LOG.error("FAILED %s", d.getDisplayName());
+ }
+
+ @Override
+ public void finished(Description d)
+ {
+ LOG.info("FINISHED %s", d.getDisplayName());
+ }
+ };
+
+ /**
+ * Reads file as utf-8 string and replace %%DATASOURCE%% with the provide
datasource value.
+ */
+ public String getStringFromFileAndReplaceDatasource(String filePath, String
datasource)
+ {
+ String fileString;
+ try {
+ InputStream is =
AbstractITBatchIndexTest.class.getResourceAsStream(filePath);
+ fileString = IOUtils.toString(is, StandardCharsets.UTF_8);
+ }
+ catch (IOException e) {
+ throw new ISE(e, "could not read query file: %s", filePath);
+ }
+
+ fileString = StringUtils.replace(
+ fileString,
+ "%%DATASOURCE%%",
+ datasource
+ );
+
+ return fileString;
+ }
+
+ /**
+ * Reads native queries from a file and runs against the provided datasource.
+ */
+ protected void doTestQuery(String dataSource, String queryFilePath)
+ {
+ try {
+ String query = getStringFromFileAndReplaceDatasource(queryFilePath,
dataSource);
+ queryHelper.testQueriesFromString(query);
+ }
+ catch (Exception e) {
+ LOG.error(e, "Error while running test query at path " + queryFilePath);
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Submits a sqlTask, waits for task completion.
+ */
+ protected void submitMSQTask(String sqlTask, String datasource, Map<String,
Object> msqContext) throws Exception
+ {
+ LOG.info("SqlTask - \n %s", sqlTask);
+
+ // Submit the tasks and wait for the datasource to get loaded
+ msqHelper.submitMsqTaskAndWaitForCompletion(
+ sqlTask,
+ msqContext
+ );
+
+ dataLoaderHelper.waitUntilDatasourceIsReady(datasource);
+ }
+
+ /**
+ * Sumits a sqlTask, waits for task completion.
+ */
+ protected void submitMSQTaskFromFile(String sqlFilePath, String datasource,
Map<String, Object> msqContext) throws Exception
+ {
+ String sqlTask = getStringFromFileAndReplaceDatasource(sqlFilePath,
datasource);
+ submitMSQTask(sqlTask, datasource, msqContext);
+ }
+
+ /**
+ * Runs a SQL ingest test.
+ *
+ * @param sqlFilePath path of file containing the sql query.
+ * @param queryFilePath path of file containing the native test queries to
be run on the ingested datasource.
+ * @param datasource name of the datasource. %%DATASOURCE%% in the sql and
queries will be replaced with this value.
+ * @param msqContext context parameters to be passed with MSQ API call.
+ */
+ protected void runMSQTaskandTestQueries(String sqlFilePath,
+ String queryFilePath,
+ String datasource,
+ Map<String, Object> msqContext
+ ) throws Exception
+ {
+ LOG.info("Starting MSQ test for sql path:%s, query path: %s]",
sqlFilePath, queryFilePath);
Review Comment:
Nit: space after first `:`: `path: %s`. Here and below.
##########
integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/IntegrationTestingConfigEx.java:
##########
@@ -353,6 +354,18 @@ public String getCloudBucket()
return getProperty("cloudBucket");
}
+ @Override
+ public String getCloudBucketwhenDeepStorageTypeIs(String storageType)
+ {
+ if (Objects.equals(storageType, "s3")) {
+ return getProperty("cloudBucket");
+ } else if (Objects.equals(storageType, "azure")) {
+ return getProperty("azureContainer");
+ } else {
Review Comment:
Should probably explicitly check for Google, then decide what to do if the
value is none of the above. Return null? Or, throw an error?
##########
integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractCloudInputSourceParallelIndexTest.java:
##########
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testsEx.indexer;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
+import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.logger.Logger;
+
+import java.io.Closeable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Function;
+
+/**
+ * Common abstract class for most cloud deep storage tests
+ */
+public abstract class AbstractCloudInputSourceParallelIndexTest extends
AbstractITBatchIndexTest
+{
+ private static final String INDEX_TASK =
"/indexer/wikipedia_cloud_index_task.json";
+ private static final String INDEX_QUERIES_RESOURCE =
"/indexer/wikipedia_index_queries.json";
+ private static final String INPUT_SOURCE_URIS_KEY = "uris";
+ private static final String INPUT_SOURCE_PREFIXES_KEY = "prefixes";
+ private static final String INPUT_SOURCE_OBJECTS_KEY = "objects";
+ private static final String WIKIPEDIA_DATA_1 = "wikipedia_index_data1.json";
+ private static final String WIKIPEDIA_DATA_2 = "wikipedia_index_data2.json";
+ private static final String WIKIPEDIA_DATA_3 = "wikipedia_index_data3.json";
+ private static final Logger LOG = new
Logger(AbstractCloudInputSourceParallelIndexTest.class);
+
+ String indexDatasource = "wikipedia_cloud_index_test_";
+
+ public static Object[][] resources()
+ {
+ return new Object[][]{
+ {new Pair<>(INPUT_SOURCE_URIS_KEY,
+ ImmutableList.of(
+ "%%INPUT_SOURCE_TYPE%%://%%BUCKET%%/%%PATH%%/" +
WIKIPEDIA_DATA_1,
+ "%%INPUT_SOURCE_TYPE%%://%%BUCKET%%/%%PATH%%/" +
WIKIPEDIA_DATA_2,
+ "%%INPUT_SOURCE_TYPE%%://%%BUCKET%%/%%PATH%%/" +
WIKIPEDIA_DATA_3
+ )
+ )},
+ {new Pair<>(INPUT_SOURCE_PREFIXES_KEY,
+ ImmutableList.of(
+ "%%INPUT_SOURCE_TYPE%%://%%BUCKET%%/%%PATH%%/"
+ )
+ )},
+ {new Pair<>(INPUT_SOURCE_OBJECTS_KEY,
+ ImmutableList.of(
+ ImmutableMap.of("bucket", "%%BUCKET%%", "path",
"%%PATH%%/" + WIKIPEDIA_DATA_1),
+ ImmutableMap.of("bucket", "%%BUCKET%%", "path",
"%%PATH%%/" + WIKIPEDIA_DATA_2),
+ ImmutableMap.of("bucket", "%%BUCKET%%", "path",
"%%PATH%%/" + WIKIPEDIA_DATA_3)
+ )
+ )}
+ };
+ }
+
+ public static List<String> fileList()
+ {
+ return Arrays.asList(WIKIPEDIA_DATA_1, WIKIPEDIA_DATA_2, WIKIPEDIA_DATA_3);
+ }
+
+ public String setInputSourceInPath(String inputSourceType, String
inputSourcePath)
+ {
+ if ("google".equals(inputSourceType)) {
+ return StringUtils.replace(
+ inputSourcePath,
+ "%%INPUT_SOURCE_TYPE%%",
+ "gs"
+ );
+ } else {
+ return StringUtils.replace(
+ inputSourcePath,
+ "%%INPUT_SOURCE_TYPE%%",
+ inputSourceType
+ );
+ }
+ }
+
+ public String getCloudPath(String inputSourceType)
+ {
+ if ("google".equals(inputSourceType)) {
+ return config.getProperty("googlePrefix");
Review Comment:
Please pull property names out as constants.
##########
integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractCloudInputSourceParallelIndexTest.java:
##########
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testsEx.indexer;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
+import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.logger.Logger;
+
+import java.io.Closeable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Function;
+
+/**
+ * Common abstract class for most cloud deep storage tests
+ */
+public abstract class AbstractCloudInputSourceParallelIndexTest extends
AbstractITBatchIndexTest
+{
+ private static final String INDEX_TASK =
"/indexer/wikipedia_cloud_index_task.json";
+ private static final String INDEX_QUERIES_RESOURCE =
"/indexer/wikipedia_index_queries.json";
+ private static final String INPUT_SOURCE_URIS_KEY = "uris";
+ private static final String INPUT_SOURCE_PREFIXES_KEY = "prefixes";
+ private static final String INPUT_SOURCE_OBJECTS_KEY = "objects";
+ private static final String WIKIPEDIA_DATA_1 = "wikipedia_index_data1.json";
+ private static final String WIKIPEDIA_DATA_2 = "wikipedia_index_data2.json";
+ private static final String WIKIPEDIA_DATA_3 = "wikipedia_index_data3.json";
+ private static final Logger LOG = new
Logger(AbstractCloudInputSourceParallelIndexTest.class);
+
+ String indexDatasource = "wikipedia_cloud_index_test_";
+
+ public static Object[][] resources()
Review Comment:
Isn't the return value `Pair<String, List<?>>[][]`?
##########
integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/indexer/AbstractS3InputSourceParallelIndexTest.java:
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testsEx.indexer;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
+import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.testing.utils.S3TestUtil;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.Closeable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.function.Function;
+
+import static junit.framework.Assert.fail;
+
+
+public abstract class AbstractS3InputSourceParallelIndexTest extends
AbstractITBatchIndexTest
+{
+ private static String indexDatasource;
+ private static S3TestUtil s3;
+ private static final String INDEX_TASK =
"/indexer/wikipedia_cloud_index_task.json";
+ private static final String INDEX_QUERIES_RESOURCE =
"/indexer/wikipedia_index_queries.json";
+ private static final String INPUT_SOURCE_URIS_KEY = "uris";
+ private static final String INPUT_SOURCE_PREFIXES_KEY = "prefixes";
+ private static final String INPUT_SOURCE_OBJECTS_KEY = "objects";
+ private static final String WIKIPEDIA_DATA_1 = "wikipedia_index_data1.json";
+ private static final String WIKIPEDIA_DATA_2 = "wikipedia_index_data2.json";
+ private static final String WIKIPEDIA_DATA_3 = "wikipedia_index_data3.json";
+
+ public static Object[][] resources()
+ {
+ return new Object[][]{
+ {new Pair<>(INPUT_SOURCE_URIS_KEY,
+ ImmutableList.of(
+ "s3://%%BUCKET%%/%%PATH%%/" + WIKIPEDIA_DATA_1,
+ "s3://%%BUCKET%%/%%PATH%%/" + WIKIPEDIA_DATA_2,
+ "s3://%%BUCKET%%/%%PATH%%/" + WIKIPEDIA_DATA_3
+ )
+ )},
+ {new Pair<>(INPUT_SOURCE_PREFIXES_KEY,
+ ImmutableList.of(
+ "s3://%%BUCKET%%/%%PATH%%/"
+ )
+ )},
+ {new Pair<>(INPUT_SOURCE_OBJECTS_KEY,
+ ImmutableList.of(
+ ImmutableMap.of("bucket", "%%BUCKET%%", "path",
"%%PATH%%/" + WIKIPEDIA_DATA_1),
+ ImmutableMap.of("bucket", "%%BUCKET%%", "path",
"%%PATH%%/" + WIKIPEDIA_DATA_2),
+ ImmutableMap.of("bucket", "%%BUCKET%%", "path",
"%%PATH%%/" + WIKIPEDIA_DATA_3)
+ )
+ )}
+ };
+ }
+
+ public static List<String> fileList()
+ {
+ return Arrays.asList(WIKIPEDIA_DATA_1, WIKIPEDIA_DATA_2, WIKIPEDIA_DATA_3);
+ }
+
+ @BeforeClass
+ public static void uploadDataFilesToS3()
+ {
+ List<String> filesToUpload = new ArrayList<>();
+ String localPath = "resources/data/batch_index/json/";
+ for (String file : fileList()) {
+ filesToUpload.add(localPath + file);
Review Comment:
It is more readable even for old-timers like me. However, I always get
feedback asking me to change loops to use streaming, which I find harder to
read. Since I'm the only one review this, let's just keep it a loop unless
someone else complains.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
