This is an automated email from the ASF dual-hosted git repository.
maytasm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new dd7a32a Fix ITSqlInputSourceTest (#10194)
dd7a32a is described below
commit dd7a32ad48e759b6e9e533c77397af3268d34e48
Author: Maytas Monsereenusorn <[email protected]>
AuthorDate: Tue Jul 21 09:52:13 2020 -0700
Fix ITSqlInputSourceTest (#10194)
* Fix ITSqlInputSourceTest.java
* Fix ITSqlInputSourceTest.java
* Fix ITSqlInputSourceTest.java
* fix
* fix
* fix
* fix
* fix
* fix
* fix
* fix
---
.travis.yml | 17 +++++++++++++++--
integration-tests/README.md | 6 +++++-
integration-tests/docker/druid.sh | 6 +++---
.../docker/environment-configs/middlemanager | 3 +--
.../test/java/org/apache/druid/tests/TestNGGroup.java | 2 ++
.../{parallelized => indexer}/ITSqlInputSourceTest.java | 17 ++++++++---------
6 files changed, 34 insertions(+), 17 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index f3b42c4..6378c72 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -321,6 +321,14 @@ jobs:
script: *run_integration_test
after_failure: *integration_test_diags
+ - &integration_input_source
+ name: "(Compile=openjdk8, Run=openjdk8) input source integration test"
+ jdk: openjdk8
+ services: *integration_test_services
+ env: TESTNG_GROUPS='-Dgroups=input-source' JVM_RUNTIME='-Djvm.runtime=8'
+ script: *run_integration_test
+ after_failure: *integration_test_diags
+
- &integration_perfect_rollup_parallel_batch_index
name: "(Compile=openjdk8, Run=openjdk8) perfect rollup parallel batch
index integration test"
jdk: openjdk8
@@ -405,7 +413,7 @@ jobs:
name: "(Compile=openjdk8, Run=openjdk8) other integration test"
jdk: openjdk8
services: *integration_test_services
- env:
TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,perfect-rollup-parallel-batch-index,kafka-index,query,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-g
[...]
+ env:
TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-sto
[...]
script: *run_integration_test
after_failure: *integration_test_diags
# END - Integration tests for Compile with Java 8 and Run with Java 8
@@ -421,6 +429,11 @@ jobs:
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=input-format' JVM_RUNTIME='-Djvm.runtime=11'
+ - <<: *integration_input_source
+ name: "(Compile=openjdk8, Run=openjdk11) input source integration test"
+ jdk: openjdk8
+ env: TESTNG_GROUPS='-Dgroups=input-source' JVM_RUNTIME='-Djvm.runtime=11'
+
- <<: *integration_perfect_rollup_parallel_batch_index
name: "(Compile=openjdk8, Run=openjdk11) perfect rollup parallel batch
index integration test"
jdk: openjdk8
@@ -449,7 +462,7 @@ jobs:
- <<: *integration_tests
name: "(Compile=openjdk8, Run=openjdk11) other integration test"
jdk: openjdk8
- env:
TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,perfect-rollup-parallel-batch-index,kafka-index,query,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-g
[...]
+ env:
TESTNG_GROUPS='-DexcludedGroups=batch-index,input-format,input-source,perfect-rollup-parallel-batch-index,kafka-index,query,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-sto
[...]
# END - Integration tests for Compile with Java 8 and Run with Java 11
- name: "security vulnerabilities"
diff --git a/integration-tests/README.md b/integration-tests/README.md
index 8157826..4c96a48 100644
--- a/integration-tests/README.md
+++ b/integration-tests/README.md
@@ -374,7 +374,11 @@ By default, test methods in a test class will be run in
sequential order one at
class can be set to run in parallel (multiple test methods of each class
running at the same time) by excluding
the given class/package from the "AllSerializedTests" test tag section and
including it in the "AllParallelizedTests"
test tag section in integration-tests/src/test/resources/testng.xml. TestNG
uses two parameters, i.e.,
-`thread-count` and `data-provider-thread-count`, for parallel test execution,
which are set to 2 for Druid integration tests.
+`thread-count` and `data-provider-thread-count`, for parallel test execution,
which are both set to 2 for Druid integration tests.
+For test using parallel execution with data provider, you will also need to
set `@DataProvider(parallel = true)`
+on your data provider method in your test class. Note that for test using
parallel execution with data provider, the test
+class does not need to be in the "AllParallelizedTests" test tag section and
if it is in the "AllParallelizedTests"
+test tag section it will actually be run with `thread-count` times
`data-provider-thread-count` threads.
You may want to modify those values for faster execution.
See https://testng.org/doc/documentation-main.html#parallel-running and
https://testng.org/doc/documentation-main.html#parameters-dataproviders for
details.
Please be mindful when adding tests to the "AllParallelizedTests" test tag
that the tests can run in parallel with
diff --git a/integration-tests/docker/druid.sh
b/integration-tests/docker/druid.sh
index 0c06838..a17df11 100755
--- a/integration-tests/docker/druid.sh
+++ b/integration-tests/docker/druid.sh
@@ -95,11 +95,11 @@ setupData()
fi
- # The SqlInputSource tests in the "batch-index" test group require data to
be setup in MySQL before running the tests.
- if [ "$DRUID_INTEGRATION_TEST_GROUP" = "batch-index" ] ; then
+ # The SqlInputSource tests in the "input-source" test group require data to
be setup in MySQL before running the tests.
+ if [ "$DRUID_INTEGRATION_TEST_GROUP" = "input-source" ] ; then
# touch is needed because OverlayFS's copy-up operation breaks POSIX
standards. See https://github.com/docker/for-linux/issues/72.
find /var/lib/mysql -type f -exec touch {} \; && service mysql start \
- && echo "CREATE database sqlinputsource DEFAULT CHARACTER SET
utf8mb4;" | mysql -u root druid \
+ && echo "GRANT ALL ON sqlinputsource.* TO 'druid'@'%'; CREATE database
sqlinputsource DEFAULT CHARACTER SET utf8mb4;" | mysql -u root druid \
&& cat /test-data/sql-input-source-sample-data.sql | mysql -u root
druid \
&& /etc/init.d/mysql stop
fi
diff --git a/integration-tests/docker/environment-configs/middlemanager
b/integration-tests/docker/environment-configs/middlemanager
index 0ca4dbc..c92cfd7 100644
--- a/integration-tests/docker/environment-configs/middlemanager
+++ b/integration-tests/docker/environment-configs/middlemanager
@@ -25,7 +25,6 @@ SERVICE_DRUID_JAVA_OPTS=-server -Xmx64m -Xms64m -XX:+UseG1GC
-agentlib:jdwp=tran
# Druid configs
druid_server_http_numThreads=100
-druid_worker_capacity=3
druid_storage_storageDirectory=/shared/storage
druid_indexer_runner_javaOptsArray=["-server", "-Xmx256m", "-Xms256m",
"-XX:NewSize=128m", "-XX:MaxNewSize=128m", "-XX:+UseG1GC",
"-Duser.timezone=UTC", "-Dfile.encoding=UTF-8",
"-Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml"]
@@ -37,4 +36,4 @@ druid_indexer_task_chathandler_type=announce
druid_auth_basic_common_cacheDirectory=/tmp/authCache/middleManager
druid_startup_logging_logProperties=true
druid_server_https_crlPath=/tls/revocations.crl
-druid_worker_capacity=20
\ No newline at end of file
+druid_worker_capacity=20
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java
b/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java
index ead9f13..d759ff2 100644
--- a/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java
+++ b/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java
@@ -29,6 +29,8 @@ public class TestNGGroup
public static final String INPUT_FORMAT = "input-format";
+ public static final String INPUT_SOURCE = "input-source";
+
public static final String KAFKA_INDEX = "kafka-index";
public static final String KAFKA_INDEX_SLOW = "kafka-index-slow";
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/parallelized/ITSqlInputSourceTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITSqlInputSourceTest.java
similarity index 86%
rename from
integration-tests/src/test/java/org/apache/druid/tests/parallelized/ITSqlInputSourceTest.java
rename to
integration-tests/src/test/java/org/apache/druid/tests/indexer/ITSqlInputSourceTest.java
index 4efe83a..92e9db8 100644
---
a/integration-tests/src/test/java/org/apache/druid/tests/parallelized/ITSqlInputSourceTest.java
+++
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITSqlInputSourceTest.java
@@ -17,14 +17,13 @@
* under the License.
*/
-package org.apache.druid.tests.parallelized;
+package org.apache.druid.tests.indexer;
import com.google.common.collect.ImmutableList;
import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.testing.guice.DruidTestModuleFactory;
import org.apache.druid.tests.TestNGGroup;
-import org.apache.druid.tests.indexer.AbstractITBatchIndexTest;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@@ -34,23 +33,23 @@ import java.util.List;
import java.util.UUID;
import java.util.function.Function;
-@Test(groups = TestNGGroup.BATCH_INDEX)
+@Test(groups = TestNGGroup.INPUT_SOURCE)
@Guice(moduleFactory = DruidTestModuleFactory.class)
public class ITSqlInputSourceTest extends AbstractITBatchIndexTest
{
- private static final String INDEX_TASK =
"/indexer/wikipedia_index_task.json";
+ private static final String INDEX_TASK =
"/indexer/wikipedia_parallel_index_using_sqlinputsource_task.json";
private static final String INDEX_QUERIES_RESOURCE =
"/indexer/wikipedia_index_queries.json";
- @DataProvider(parallel = true)
+ @DataProvider
public static Object[][] resources()
{
return new Object[][]{
// Multiple query. No filter
{ImmutableList.of("SELECT * FROM wikipedia_index_data1", "SELECT *
FROM wikipedia_index_data2", "SELECT * FROM wikipedia_index_data3")},
// Multiple query. Filter on timestamp column
- {ImmutableList.of("SELECT * FROM wikipedia_index_data1 WHERE timestamp
BETWEEN '2013-08-31 00:00:00' AND '2013-08-31 11:59:59'",
- "SELECT * FROM wikipedia_index_data2 WHERE timestamp
BETWEEN '2013-08-31 00:00:00' AND '2013-09-01 11:59:59'",
- "SELECT * FROM wikipedia_index_data3 WHERE timestamp
BETWEEN '2013-09-01 00:00:00' AND '2013-09-01 11:59:59'")},
+ {ImmutableList.of("SELECT * FROM wikipedia_index_data1 WHERE timestamp
BETWEEN '2013-08-31 00:00:00' AND '2013-09-02 00:00:00'",
+ "SELECT * FROM wikipedia_index_data2 WHERE timestamp
BETWEEN '2013-08-31 00:00:00' AND '2013-09-02 00:00:00'",
+ "SELECT * FROM wikipedia_index_data3 WHERE timestamp
BETWEEN '2013-09-01 00:00:00' AND '2013-09-02 00:00:00'")},
// Multiple query. Filter on data column
{ImmutableList.of("SELECT * FROM wikipedia_index_data1 WHERE added >
0",
"SELECT * FROM wikipedia_index_data2 WHERE added >
0",
@@ -58,7 +57,7 @@ public class ITSqlInputSourceTest extends
AbstractITBatchIndexTest
// Single query. No filter
{ImmutableList.of("SELECT * FROM wikipedia_index_data_all")},
// Single query. Filter on timestamp column
- {ImmutableList.of("SELECT * FROM wikipedia_index_data_all WHERE
timestamp BETWEEN '2013-08-31 00:00:00' AND '2013-09-01 11:59:59'")},
+ {ImmutableList.of("SELECT * FROM wikipedia_index_data_all WHERE
timestamp BETWEEN '2013-08-31 00:00:00' AND '2013-09-02 00:00:00'")},
// Single query. Filter on data column
{ImmutableList.of("SELECT * FROM wikipedia_index_data_all WHERE added
> 0")},
};
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
