This is an automated email from the ASF dual-hosted git repository.
karan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new a04e2080ce4 Split QTest runs into multiple splits (#18173)
a04e2080ce4 is described below
commit a04e2080ce411f59e0a018c9d09dfd3ca9952f5c
Author: Zoltan Haindrich <[email protected]>
AuthorDate: Fri Jun 27 15:01:58 2025 +0200
Split QTest runs into multiple splits (#18173)
* disable jacoco on pushes
* wondering
* Revert "wondering"
This reverts commit 22a82e335460795ecec947a55d38b5d67b51dbd1.
* add properties
* Reapply "wondering"
This reverts commit 1febbf4c085a079003a76e86b7c5d1fb044d0149.
* add quidem.split
---
.github/workflows/ci.yml | 10 +-
.../apache/druid/quidem/DruidQuidemTestBase.java | 106 +++++++++++++++++----
2 files changed, 95 insertions(+), 21 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ca5229780c9..b8bdd7ba157 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -59,7 +59,7 @@ jobs:
name: "coverage-jacoco"
needs: run-unit-tests
uses: ./.github/workflows/worker.yml
- if: ${{ !contains( github.event.pull_request.labels.*.name, 'jacoco:skip')
}}
+ if: ${{ !cancelled() && !contains(
github.event.pull_request.labels.*.name, 'jacoco:skip') && github.event_name !=
'push' }}
with:
script: .github/scripts/create-jacoco-coverage-report.sh
artifacts_to_download: "unit-test-reports-*"
@@ -82,8 +82,12 @@ jobs:
# this will be running in parallel with the ITs later; but until that
migration happens - run them in parallel with normal tests
run-qtest:
+ strategy:
+ fail-fast: false
+ matrix:
+ split: [ "0/4", "1/4", "2/4", "3/4" ]
uses: ./.github/workflows/worker.yml
with:
- script: .github/scripts/run_unit-tests -Dtest=QTest -fae
- key: quidem-QTest
+ script: .github/scripts/run_unit-tests -Dtest=QTest -Dquidem.split=${{
matrix.split }} -fae
+ key: QTest-${{ matrix.split }}
jdk: 17
diff --git a/sql/src/test/java/org/apache/druid/quidem/DruidQuidemTestBase.java
b/sql/src/test/java/org/apache/druid/quidem/DruidQuidemTestBase.java
index f2e9c515e23..2215c7670e7 100644
--- a/sql/src/test/java/org/apache/druid/quidem/DruidQuidemTestBase.java
+++ b/sql/src/test/java/org/apache/druid/quidem/DruidQuidemTestBase.java
@@ -19,6 +19,7 @@
package org.apache.druid.quidem;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.io.Files;
@@ -29,6 +30,7 @@ import net.hydromatic.quidem.Quidem.ConfigBuilder;
import org.apache.calcite.test.DiffTestCase;
import org.apache.calcite.util.Closer;
import org.apache.calcite.util.Util;
+import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.druid.concurrent.Threads;
import org.apache.druid.error.DruidException;
import org.apache.druid.java.util.common.FileUtils;
@@ -44,7 +46,6 @@ import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
-import javax.annotation.Nullable;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -61,6 +62,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import static org.junit.jupiter.api.Assertions.fail;
@@ -100,7 +103,12 @@ public abstract class DruidQuidemTestBase
private static final String PROPERTY_FILTER = "quidem.filter";
- private final String filterStr;
+ /**
+ * This property enables the test system to split up huge cases into desired
+ * number of smaller testcases.
+ */
+ private static final String PROPERTY_SPLIT = "quidem.split";
+
private final PathMatcher filterMatcher;
private DruidQuidemRunner druidQuidemRunner;
@@ -112,30 +120,92 @@ public abstract class DruidQuidemTestBase
public DruidQuidemTestBase(DruidQuidemRunner druidQuidemRunner)
{
- this.filterStr = System.getProperty(PROPERTY_FILTER, null);
- this.filterMatcher = buildFilterMatcher(filterStr);
+ String filterStr = Strings.emptyToNull(System.getProperty(PROPERTY_FILTER,
null));
+ String splitStr = Strings.emptyToNull(System.getProperty(PROPERTY_SPLIT,
null));
+ this.filterMatcher = buildFilterMatcher(filterStr, splitStr);
this.druidQuidemRunner = druidQuidemRunner;
}
- private static PathMatcher buildFilterMatcher(@Nullable String filterStr)
+ private PathMatcher buildFilterMatcher(String filterStr, String splitStr)
+ {
+ if (filterStr != null && splitStr != null) {
+ throw new IAE(
+ "Cannot configure multiple filter methods with properties: %s and
%s.", PROPERTY_FILTER, PROPERTY_SPLIT
+ );
+ }
+ if (filterStr != null) {
+ return new IQPathMatcher(filterStr);
+ }
+ if (splitStr != null) {
+ return new QuidemSplitPathMatcher(splitStr);
+ }
+ return TrueFileFilter.INSTANCE;
+ }
+
+ static class QuidemSplitPathMatcher implements PathMatcher
+ {
+ private final int splitIndex;
+ private final int splitCount;
+
+ public QuidemSplitPathMatcher(String splitStr)
+ {
+ Pattern pattern = Pattern.compile("^([0-9]+)/([0-9]+)$");
+ Matcher m = pattern.matcher(splitStr);
+ if (!m.matches()) {
+ throw DruidException.defensive("Invalid split pattern; must match
pattern [%s]", pattern);
+ }
+ splitIndex = Integer.parseInt(m.group(1));
+ splitCount = Integer.parseInt(m.group(2));
+ if (splitCount < 1 || splitIndex < 0 || splitIndex >= splitCount) {
+ throw DruidException.defensive("invalid splitStr [%s]", splitStr);
+ }
+ }
+
+ @Override
+ public boolean matches(Path path)
+ {
+ return Math.floorMod(path.toString().hashCode(), splitCount) ==
splitIndex;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "split:" + splitIndex + "/" + splitCount;
+ }
+ }
+
+ static class IQPathMatcher implements PathMatcher
{
- if (null == filterStr) {
- return f -> true;
+ private final List<PathMatcher> filterMatchers = new ArrayList<>();
+ private final String filterStr;
+
+ public IQPathMatcher(String filterStr)
+ {
+ this.filterStr = filterStr;
+ final FileSystem fileSystem = FileSystems.getDefault();
+ for (String filterGlob : filterStr.split(",")) {
+ if (!filterGlob.endsWith("*") && !filterGlob.endsWith(IQ_SUFFIX)) {
+ filterGlob = filterStr + IQ_SUFFIX;
+ }
+ filterMatchers.add(fileSystem.getPathMatcher("glob:" + filterGlob));
+ }
}
- final FileSystem fileSystem = FileSystems.getDefault();
- final List<PathMatcher> filterMatchers = new ArrayList<>();
- for (String filterGlob : filterStr.split(",")) {
- if (!filterGlob.endsWith("*") && !filterGlob.endsWith(IQ_SUFFIX)) {
- filterGlob = filterStr + IQ_SUFFIX;
+ @Override
+ public boolean matches(Path path)
+ {
+ for (PathMatcher m : filterMatchers) {
+ if (m.matches(path)) {
+ return true;
+ }
}
- filterMatchers.add(fileSystem.getPathMatcher("glob:" + filterGlob));
+ return false;
}
- if (filterMatchers.isEmpty()) {
- return f -> true;
- } else {
- return f -> filterMatchers.stream().anyMatch(m -> m.matches(f));
+ @Override
+ public String toString()
+ {
+ return filterStr;
}
}
@@ -372,7 +442,7 @@ public abstract class DruidQuidemTestBase
throw new IAE(
"There are no test cases in directory[%s] or there are no matches to
filter[%s]",
testRoot,
- filterStr
+ filterMatcher
);
}
Collections.sort(ret);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
