This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 66f7770fcec Various CI enhancements (#18893)
66f7770fcec is described below
commit 66f7770fcec8d36c82552fbee2a90ff4fdccf5d1
Author: Zoltan Haindrich <[email protected]>
AuthorDate: Tue Jan 13 10:19:12 2026 +0100
Various CI enhancements (#18893)
* move static-checks to separate scripts (runnable locally) + use
`worker.yml` - this is to prepare them to be run them alongside with the main ci
* remove now not necessary `build` stage as we are down to just a few old
style integration tests
* feels like creating a report at the run level doesn't fully match how
people expect it to behave...creating one in every split I guess will feel more
straightforward
* reorg a bit how additional tests are run
* default to run things with jdk21
* fix jacoco to not load 2 different versions
* upgrade commons-compress to 1.27.0 to run all the unit-tests/etc
* skip jdk17 unless `test-jdk17` label is present
---
.github/scripts/analyze_dependencies_script.sh | 2 +-
.github/scripts/license_checks_script.sh | 2 +-
.../{license_checks_script.sh => openrewrite.sh} | 20 +--
...license_checks_script.sh => packaging-check.sh} | 16 +-
...{test_balancing_calc => static-checks-maven.sh} | 34 ++--
...ense_checks_script.sh => strict-compilation.sh} | 14 +-
.github/scripts/test_balancing_calc | 46 ++++-
.../{license_checks_script.sh => web-checks.sh} | 35 +++-
.github/workflows/ci.yml | 55 ++----
.github/workflows/reusable-standard-its.yml | 8 -
.github/workflows/static-checks.yml | 193 +++------------------
.../unit-and-integration-tests-unified.yml | 56 +-----
.github/workflows/worker.yml | 24 ++-
check_test_suite.py | 192 --------------------
check_test_suite_test.py | 154 ----------------
licenses.yaml | 2 +-
pom.xml | 2 +-
17 files changed, 184 insertions(+), 671 deletions(-)
diff --git a/.github/scripts/analyze_dependencies_script.sh
b/.github/scripts/analyze_dependencies_script.sh
index 1212dbd1cd2..9da6c05d778 100755
--- a/.github/scripts/analyze_dependencies_script.sh
+++ b/.github/scripts/analyze_dependencies_script.sh
@@ -15,7 +15,7 @@
#!bin/bash
-${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true
-DignoreNonCompile=true -DfailOnWarning=true ||
+mvn -B dependency:analyze -DoutputXML=true -DignoreNonCompile=true
-DfailOnWarning=true ||
{ echo "
The dependency analysis has found a dependency that is either:
diff --git a/.github/scripts/license_checks_script.sh
b/.github/scripts/license_checks_script.sh
index 163214f8298..e278fb4a7fa 100755
--- a/.github/scripts/license_checks_script.sh
+++ b/.github/scripts/license_checks_script.sh
@@ -18,7 +18,7 @@
set -e
./.github/scripts/setup_generate_license.sh
-${MVN} apache-rat:check -Prat --fail-at-end \
+mvn -B apache-rat:check -Prat --fail-at-end \
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
\
-Drat.consoleOutput=true
# Generate dependency reports and checks they are valid.
diff --git a/.github/scripts/license_checks_script.sh
b/.github/scripts/openrewrite.sh
similarity index 63%
copy from .github/scripts/license_checks_script.sh
copy to .github/scripts/openrewrite.sh
index 163214f8298..af0173a43d1 100755
--- a/.github/scripts/license_checks_script.sh
+++ b/.github/scripts/openrewrite.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,15 +15,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-#!/bin/bash
-
set -e
+set -x
+
+export MAVEN_OPTS=-Xmx8g
+
+echo 'Running Maven install...'
+mvn -B clean install -q -ff -pl '!distribution' -P skip-tests
-Dweb.console.skip=true -T1C
+mvn -B install -q -ff -pl 'distribution' -P skip-tests -Dweb.console.skip=true
-./.github/scripts/setup_generate_license.sh
-${MVN} apache-rat:check -Prat --fail-at-end \
--Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
\
--Drat.consoleOutput=true
-# Generate dependency reports and checks they are valid.
-mkdir -p target
-distribution/bin/generate-license-dependency-reports.py . target
--clean-maven-artifact-transfer --parallel 2
-distribution/bin/check-licenses.py licenses.yaml target/license-reports
+mvn -B rewrite:dryRun -Dweb.console.skip=true
diff --git a/.github/scripts/license_checks_script.sh
b/.github/scripts/packaging-check.sh
similarity index 66%
copy from .github/scripts/license_checks_script.sh
copy to .github/scripts/packaging-check.sh
index 163214f8298..fe9ec3eb0e1 100755
--- a/.github/scripts/license_checks_script.sh
+++ b/.github/scripts/packaging-check.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,15 +15,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-#!/bin/bash
-
set -e
+set -x
./.github/scripts/setup_generate_license.sh
-${MVN} apache-rat:check -Prat --fail-at-end \
--Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
\
--Drat.consoleOutput=true
-# Generate dependency reports and checks they are valid.
-mkdir -p target
-distribution/bin/generate-license-dependency-reports.py . target
--clean-maven-artifact-transfer --parallel 2
-distribution/bin/check-licenses.py licenses.yaml target/license-reports
+mvn -B clean install -Prat --fail-at-end \
+ -pl '!benchmarks, !distribution' -P skip-tests -Dweb.console.skip=false -T1C
+mvn -B install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end \
+ -pl 'distribution' -P skip-tests -Dweb.console.skip=false -T1C
diff --git a/.github/scripts/test_balancing_calc
b/.github/scripts/static-checks-maven.sh
similarity index 54%
copy from .github/scripts/test_balancing_calc
copy to .github/scripts/static-checks-maven.sh
index f619278b106..0b643a8b3df 100755
--- a/.github/scripts/test_balancing_calc
+++ b/.github/scripts/static-checks-maven.sh
@@ -14,25 +14,27 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#--------------------------------------------------------------------
-
-# Helps to calculate new balancing
-# on the output a simple knapsack should be done...right now that's manual
set -e
+set -x
+
+echo 'Running Maven install...'
+mvn -B clean install -q -ff -pl '!distribution' -P skip-tests
-Dweb.console.skip=true -Dmaven.javadoc.skip=true -T1C
+mvn -B install -q -ff -pl 'distribution' -P skip-tests -Dweb.console.skip=true
-Dmaven.javadoc.skip=true
+
+mvn -B checkstyle:checkstyle --fail-at-end
+
+./.github/scripts/license_checks_script.sh
+
+./.github/scripts/analyze_dependencies_script.sh
+
+mvn -B animal-sniffer:check --fail-at-end
-find . -name 'TEST*xml' |
-# <testsuite ... name="org.apache.druid.query.policy.NoRestrictionPolicyTest"
time="0.071" ...>
-xargs sed -nr
'/^<testsuite/s/.*name=\"([^\"]+)\".*time=\"([^\"]+)\".*/\1\t\2/p' |
-# org.apache.druid.server.RequestLogLineTest 0.052
-sed -r 's/[^\t]+\.(.)[^\t.]+\t/\1\t/' > test_times
+mvn -B enforcer:enforce --fail-at-end
-for c in {A..Z} ;do
- echo -n "$c ";
- (echo 0;grep "^$c" test_times| cut -f 2)|paste -s -d + -|bc
-done | sort -k 2 -nr
-# final output is a table like
-# <letter> <sumTime>
-# C 88
+mvn -B forbiddenapis:check forbiddenapis:testCheck --fail-at-end
+# TODO: consider adding pmd:cpd-check
+mvn -B pmd:check --fail-at-end
+mvn -B spotbugs:check --fail-at-end -pl '!benchmarks'
diff --git a/.github/scripts/license_checks_script.sh
b/.github/scripts/strict-compilation.sh
similarity index 63%
copy from .github/scripts/license_checks_script.sh
copy to .github/scripts/strict-compilation.sh
index 163214f8298..1978089f86b 100755
--- a/.github/scripts/license_checks_script.sh
+++ b/.github/scripts/strict-compilation.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,15 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-#!/bin/bash
-
set -e
+set -x
-./.github/scripts/setup_generate_license.sh
-${MVN} apache-rat:check -Prat --fail-at-end \
--Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
\
--Drat.consoleOutput=true
-# Generate dependency reports and checks they are valid.
-mkdir -p target
-distribution/bin/generate-license-dependency-reports.py . target
--clean-maven-artifact-transfer --parallel 2
-distribution/bin/check-licenses.py licenses.yaml target/license-reports
+mvn -B clean -DstrictCompile -pl '!embedded-tests' compile test-compile
--fail-at-end -P skip-tests -Dweb.console.skip=true -T1C
diff --git a/.github/scripts/test_balancing_calc
b/.github/scripts/test_balancing_calc
index f619278b106..bbd4c6ec824 100755
--- a/.github/scripts/test_balancing_calc
+++ b/.github/scripts/test_balancing_calc
@@ -28,11 +28,53 @@ xargs sed -nr
'/^<testsuite/s/.*name=\"([^\"]+)\".*time=\"([^\"]+)\".*/\1\t\2/p'
sed -r 's/[^\t]+\.(.)[^\t.]+\t/\1\t/' > test_times
for c in {A..Z} ;do
- echo -n "$c ";
+ echo -n "$c ";
(echo 0;grep "^$c" test_times| cut -f 2)|paste -s -d + -|bc
-done | sort -k 2 -nr
+done | sed -r 's/ +/\t/' | sort -k 2 -nr > letter_times
+
# final output is a table like
# <letter> <sumTime>
# C 88
+echo "=== Letter times (sorted by duration) ==="
+cat letter_times
+
+NUM_BUCKETS=8
+
+echo ""
+echo "=== Balanced buckets ($NUM_BUCKETS buckets) ==="
+
+# Initialize buckets
+declare -a bucket_time
+declare -a bucket_letters
+for (( i=0; i<NUM_BUCKETS; i++ )); do
+ bucket_time[$i]=0
+ bucket_letters[$i]=""
+done
+
+# Greedy bin packing: assign each letter to the bucket with the smallest total
time
+while IFS=$'\t' read -r letter time; do
+ # Find bucket with minimum time
+ min_idx=0
+ min_time=${bucket_time[0]}
+ for (( i=1; i<NUM_BUCKETS; i++ )); do
+ if (( $(echo "${bucket_time[$i]} < $min_time" | bc -l) )); then
+ min_time=${bucket_time[$i]}
+ min_idx=$i
+ fi
+ done
+
+ # Add letter to that bucket
+
bucket_letters[$min_idx]="${bucket_letters[$min_idx]:+${bucket_letters[$min_idx]},}$letter*"
+ bucket_time[$min_idx]=$(echo "${bucket_time[$min_idx]} + $time" | bc -l)
+done < letter_times
+
+# Output buckets sorted by time
+for (( i=0; i<NUM_BUCKETS; i++ )); do
+ echo "${bucket_time[$i]}"$'\t'"${bucket_letters[$i]}"$'\t'"bucket-$i"
+done | sort -k 1 -nr
+echo -n "suggested pattern: "
+for (( i=0; i<NUM_BUCKETS; i++ )); do
+ echo -n "\"${bucket_letters[$i]}\", "
+done
diff --git a/.github/scripts/license_checks_script.sh
b/.github/scripts/web-checks.sh
similarity index 57%
copy from .github/scripts/license_checks_script.sh
copy to .github/scripts/web-checks.sh
index 163214f8298..353041baa53 100755
--- a/.github/scripts/license_checks_script.sh
+++ b/.github/scripts/web-checks.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,15 +15,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-#!/bin/bash
-
set -e
+set -x
+
+# Install node.js via maven frontend plugin (same approach as jacoco script)
+mvn -B
com.github.eirslett:frontend-maven-plugin:install-node-and-npm@install-node-and-npm
-pl web-console/
+PATH+=:web-console/target/node/
+
+# docs
+(cd website && npm install)
+cd website
+npm run build
+npm run link-lint
+npm run spellcheck
+cd ..
+
+# web console
+mvn -B test -pl 'web-console'
+cd web-console
+{ for i in 1 2 3; do npm run codecov && break || sleep 15; done }
+cd ..
+# web console end-to-end test
./.github/scripts/setup_generate_license.sh
-${MVN} apache-rat:check -Prat --fail-at-end \
--Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
\
--Drat.consoleOutput=true
-# Generate dependency reports and checks they are valid.
-mkdir -p target
-distribution/bin/generate-license-dependency-reports.py . target
--clean-maven-artifact-transfer --parallel 2
-distribution/bin/check-licenses.py licenses.yaml target/license-reports
+web-console/script/druid build
+web-console/script/druid start
+(cd web-console && npm run test-e2e)
+web-console/script/druid stop
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 661213f17f6..99997a76f0c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -25,13 +25,14 @@ jobs:
fail-fast: false
matrix:
jdk: [ "17", "21" ]
- pattern: [ "C*", "H*,U*,V*", "N*,Q*,S*", "B*,O*,R*", "G*,J*,K*",
"F*,L*,M*", "A*,D*,I*,X*,Y*,Z*", "E*,P*,T*,W*"]
+ pattern: [ "C*", "D*,H*,N*,L*", "I*,A*,U*", "K*,E*,W*,Z*,Y*,X*",
"M*,P*,O*", "R*,B*,Q*,V*", "S*", "T*,F*,G*,J*" ]
uses: ./.github/workflows/worker.yml
with:
script: .github/scripts/run_unit-tests -Dtest=!QTest,'${{ matrix.pattern
}}' -Dmaven.test.failure.ignore=true
artifact_prefix: "unit-test-reports-jdk${{ matrix.jdk }}"
jdk: ${{ matrix.jdk }}
key: "test-jdk${{ matrix.jdk }}-[${{ matrix.pattern }}]"
+ execute: ${{ matrix.jdk == '21' || github.event_name == 'push' ||
contains(github.event.pull_request.labels.*.name, 'test-jdk17') }}
validate-dist:
uses: ./.github/workflows/worker.yml
@@ -39,51 +40,31 @@ jobs:
script: .github/scripts/validate-dist
key: "validate-dist"
- test-report:
- name: "test-report"
- needs: run-unit-tests
- runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- matrix:
- jdk: [ "17", "21" ]
- steps:
- - name: Download reports for all unit test jobs
- uses: actions/download-artifact@v4
- with:
- pattern: "unit-test-reports-jdk${{ matrix.jdk }}*"
- path: target/surefire-reports
-
- - name: Publish Test Report
- uses: mikepenz/action-junit-report@v5
- with:
- check_name: "Unit Test Report (JDK ${{ matrix.jdk }})"
- report_paths: '**/target/surefire-reports/TEST-*.xml'
- detailed_summary: true
- flaky_summary: true
- annotate_only: true
- fail_on_failure: true
- check_retries: true
- truncate_stack_traces: false
-
reporting-jacoco-coverage-failures:
name: "coverage-jacoco"
- needs: run-unit-tests
+ needs: [run-unit-tests, run-separated-tests]
uses: ./.github/workflows/worker.yml
- if: ${{ !cancelled() && !contains(
github.event.pull_request.labels.*.name, 'jacoco:skip') && github.event_name !=
'push' }}
+ if: ${{ success() && !contains( github.event.pull_request.labels.*.name,
'jacoco:skip') && github.event_name != 'push' }}
with:
script: .github/scripts/create-jacoco-coverage-report.sh
- artifacts_to_download: "unit-test-reports-*"
+ artifacts_to_download: "unit-test-reports-jdk21*"
key: "coverage-jacoco"
- # this will be running in parallel with the ITs later; but until that
migration happens - run them in parallel with normal tests
- run-qtest:
+ run-separated-tests:
strategy:
fail-fast: false
matrix:
- split: [ "0/4", "1/4", "2/4", "3/4" ]
+ include:
+ - name: "QTest 0/4"
+ args: "-Dtest=QTest -Dquidem.split=0/4"
+ - name: "QTest 1/4"
+ args: "-Dtest=QTest -Dquidem.split=1/4"
+ - name: "QTest 2/4"
+ args: "-Dtest=QTest -Dquidem.split=2/4"
+ - name: "QTest 3/4"
+ args: "-Dtest=QTest -Dquidem.split=3/4"
uses: ./.github/workflows/worker.yml
with:
- script: .github/scripts/run_unit-tests -Dtest=QTest -Dquidem.split=${{
matrix.split }} -fae
- key: QTest-${{ matrix.split }}
- jdk: 21
+ script: .github/scripts/run_unit-tests ${{ matrix.args }}
-Dmaven.test.failure.ignore=true
+ key: ${{ matrix.name }}
+ artifact_prefix: "unit-test-reports-jdk21"
diff --git a/.github/workflows/reusable-standard-its.yml
b/.github/workflows/reusable-standard-its.yml
index b23895dc2ac..1f53be2e6f4 100644
--- a/.github/workflows/reusable-standard-its.yml
+++ b/.github/workflows/reusable-standard-its.yml
@@ -68,15 +68,7 @@ jobs:
java-version: ${{ inputs.runtime_jdk }}
distribution: 'zulu'
- - name: Restore Maven repository
- id: maven-restore
- uses: actions/cache/restore@v4
- with:
- path: ~/.m2/repository
- key: maven-${{ runner.os }}-${{ inputs.build_jdk }}-${{ github.sha }}
-
- name: Maven build
- if: steps.maven-restore.outputs.cache-hit != 'true'
run: |
./it.sh ci
diff --git a/.github/workflows/static-checks.yml
b/.github/workflows/static-checks.yml
index 76e74381cc5..611f80634a2 100644
--- a/.github/workflows/static-checks.yml
+++ b/.github/workflows/static-checks.yml
@@ -30,179 +30,40 @@ concurrency:
group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label ||
github.head_ref || github.run_id }}' # group workflows only on pull_requests
and not on branch commits
cancel-in-progress: true
-env:
- MVN: mvn -B
- MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true
-Dmaven.javadoc.skip=true
- MAVEN_SKIP_TESTS: -P skip-tests
- MAVEN_OPTS: -Xmx8g
-
jobs:
- static-checks:
+ packaging-check:
strategy:
fail-fast: false
matrix:
- java: [ '17', '21' ]
- runs-on: ubuntu-latest
- steps:
- - name: checkout branch
- uses: actions/checkout@v4
-
- - uses: actions/setup-java@v4
- with:
- distribution: 'zulu'
- java-version: ${{ matrix.java }}
- cache: 'maven'
-
- - name: packaging check
- run: |
- ./.github/scripts/setup_generate_license.sh
- ${MVN} clean install -Prat --fail-at-end \
- -pl '!benchmarks, !distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
-Dweb.console.skip=false -T1C
- ${MVN} install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end \
- -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
-Dweb.console.skip=false -T1C
-
- - name: script checks
- # who watches the watchers?
- if: ${{ matrix.java == '17' }}
- run: ./check_test_suite_test.py
-
- - name: maven install
- if: ${{ matrix.java == '17' }}
- run: |
- echo 'Running Maven install...' &&
- ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP}
${MAVEN_SKIP_TESTS} -T1C &&
- ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP}
${MAVEN_SKIP_TESTS}
-
- - name: checkstyle
- if: ${{ matrix.java == '17' }}
- run: ${MVN} checkstyle:checkstyle --fail-at-end
-
- - name: license checks
- if: ${{ matrix.java == '17' }}
- run: ./.github/scripts/license_checks_script.sh
-
- - name: analyze dependencies
- if: ${{ matrix.java == '17' }}
- run: |
- ./.github/scripts/analyze_dependencies_script.sh
-
- - name: animal sniffer checks
- if: ${{ matrix.java == '17' }}
- run: ${MVN} animal-sniffer:check --fail-at-end
-
- - name: enforcer checks
- if: ${{ matrix.java == '17' }}
- run: ${MVN} enforcer:enforce --fail-at-end
-
- - name: forbidden api checks
- if: ${{ matrix.java == '17' }}
- run: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end
-
- - name: pmd checks
- if: ${{ matrix.java == '17' }}
- run: ${MVN} pmd:check --fail-at-end # TODO: consider adding
pmd:cpd-check
-
- - name: spotbugs checks
- if: ${{ matrix.java == '17' }}
- run: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks'
+ jdk: [ '17', '21' ]
+ uses: ./.github/workflows/worker.yml
+ with:
+ script: .github/scripts/packaging-check.sh
+ jdk: ${{ matrix.jdk }}
+ key: "packaging-check-jdk${{ matrix.jdk }}"
+
+ static-checks-maven:
+ uses: ./.github/workflows/worker.yml
+ with:
+ script: .github/scripts/static-checks-maven.sh
+ key: "static-checks-maven"
strict-compilation:
- runs-on: ubuntu-latest
- steps:
- - name: checkout branch
- uses: actions/checkout@v4
-
- - uses: actions/setup-java@v4
- with:
- distribution: 'zulu'
- java-version: '17'
- cache: 'maven'
-
- - name: (openjdk17) strict compilation
- # errorprone requires JDK 11+
- # Strict compilation requires more than 2 GB
- # safe to exclude embedded-tests, it's not in distribution
- run: ${MVN} clean -DstrictCompile -pl '!embedded-tests' compile
test-compile --fail-at-end ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C
+ uses: ./.github/workflows/worker.yml
+ with:
+ script: .github/scripts/strict-compilation.sh
+ key: "strict-compilation"
openrewrite:
- runs-on: ubuntu-latest
- steps:
- - name: checkout branch
- uses: actions/checkout@v4
-
- - uses: actions/setup-java@v4
- with:
- distribution: 'zulu'
- java-version: '21'
- cache: 'maven'
-
- - name: maven install
- run: |
- echo 'Running Maven install...' &&
- ${MVN} clean install -q -ff -pl '!distribution' ${MAVEN_SKIP}
${MAVEN_SKIP_TESTS} -T1C &&
- ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP}
${MAVEN_SKIP_TESTS}
-
- - name: rewrite:dryRun
- id: rewrite-dryRun
- run: |
- ${MVN} rewrite:dryRun ${MAVEN_SKIP}
-
- - name: Upload open rewrite patch
- if: ${{ failure() && steps.rewrite-dryRun.conclusion == 'failure' }}
- uses: actions/upload-artifact@master
- with:
- name: Rewrite patch
- path: ./target/rewrite/rewrite.patch
+ uses: ./.github/workflows/worker.yml
+ with:
+ script: .github/scripts/openrewrite.sh
+ key: "openrewrite"
+ artifact_prefix: "openrewrite-patch"
web-checks:
- strategy:
- fail-fast: false
- runs-on: ubuntu-latest
- steps:
- - name: checkout branch
- uses: actions/checkout@v4
-
- - uses: actions/setup-java@v4
- with:
- distribution: 'zulu'
- java-version: '17'
- cache: 'maven'
-
- - name: setup node
- uses: actions/setup-node@v3
- with:
- node-version: 18.0.0
-
- - name: docs
- run: |
- (cd website && npm install)
- cd website
- npm run build
- npm run link-lint
- npm run spellcheck
-
- - name: web console
- run: |
- ${MVN} test -pl 'web-console'
- cd web-console
- { for i in 1 2 3; do npm run codecov && break || sleep 15; done }
-
- - name: web console end-to-end test
- id: web-console-test
- run: |
- ./.github/scripts/setup_generate_license.sh
- web-console/script/druid build
- web-console/script/druid start
- (cd web-console && npm run test-e2e)
- web-console/script/druid stop
-
- - name: Tar druid logs
- if: ${{ failure() && steps.web-console-test.conclusion == 'failure' }}
- run: tar cvzf ./druid-logs.tgz -C
./distribution/target/apache-druid-*-SNAPSHOT/ log -C ./var druid/indexing-logs
-
- - name: Upload druid logs to GitHub
- if: ${{ failure() && steps.web-console-test.conclusion == 'failure' }}
- uses: actions/upload-artifact@master
- with:
- name: Druid logs web-checks
- path: druid-logs.tgz
+ uses: ./.github/workflows/worker.yml
+ with:
+ script: .github/scripts/web-checks.sh
+ key: "web-checks"
+ artifact_prefix: "web-checks-logs"
diff --git a/.github/workflows/unit-and-integration-tests-unified.yml
b/.github/workflows/unit-and-integration-tests-unified.yml
index f0110d4538e..57390917932 100644
--- a/.github/workflows/unit-and-integration-tests-unified.yml
+++ b/.github/workflows/unit-and-integration-tests-unified.yml
@@ -51,70 +51,20 @@ env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
jobs:
- build:
- name: "build (jdk${{ matrix.jdk }})"
- strategy:
- fail-fast: false
- matrix:
- jdk: [ '17', '21' ]
- runs-on: ubuntu-latest
- steps:
- - name: Checkout branch
- uses: actions/checkout@v4
-
- # skip the "cache: maven" step from setup-java. We explicitly use a
- # different cache key since we cannot reuse it across commits.
- - uses: actions/setup-java@v4
- with:
- distribution: 'zulu'
- java-version: ${{ matrix.jdk }}
-
- # the build step produces SNAPSHOT artifacts into the local maven
repository,
- # we include github.sha in the cache key to make it specific to that
build/jdk
- - name: Cache Maven m2 repository
- id: maven
- uses: actions/cache@v4
- with:
- path: ~/.m2/repository
- key: maven-${{ runner.os }}-${{ matrix.jdk }}-${{ github.sha }}
- restore-keys: setup-java-Linux-maven-${{ hashFiles('**/pom.xml') }}
-
- - name: Cache targets
- id: target
- uses: actions/cache@v4
- with:
- path: |
- ./**/target
- key: maven-${{ runner.os }}-${{ matrix.jdk }}-targets-${{ github.sha
}}
-
- - name: Cache image
- id: docker_container
- uses: actions/cache@v4
- with:
- key: druid-container-jdk${{ matrix.jdk }}.tar.gz-${{ github.sha }}
- path: |
- ./druid-container-jdk${{ matrix.jdk }}.tar.gz
- ./integration-tests-ex/image/target/env.sh
-
- - name: Maven build
- id: maven_build
- run: |
- ./it.sh ci
-
unit-tests:
name: "unit tests"
uses: ./.github/workflows/ci.yml
standard-its:
- needs: [build, unit-tests]
+ needs: [unit-tests]
uses: ./.github/workflows/standard-its.yml
docker-tests:
- needs: [build, unit-tests]
+ needs: [unit-tests]
uses: ./.github/workflows/docker-tests.yml
actions-timeline:
- needs: [build, unit-tests, standard-its, docker-tests]
+ needs: [unit-tests, standard-its, docker-tests]
runs-on: ubuntu-latest
if: ${{ !cancelled() }}
steps:
diff --git a/.github/workflows/worker.yml b/.github/workflows/worker.yml
index 676b7bbfd61..600287dded7 100644
--- a/.github/workflows/worker.yml
+++ b/.github/workflows/worker.yml
@@ -21,7 +21,7 @@ on:
jdk:
required: false
type: string
- default: '17'
+ default: '21'
script:
required: true
type: string
@@ -38,6 +38,11 @@ on:
required: false
type: string
description: "Artifacts to download. For example:
'unit-test-reports-*'"
+ execute:
+ required: false
+ type: boolean
+ default: true
+ description: 'Controls execution of the invocation - this is a
workaround for github actions matrix element skip limitation'
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
@@ -54,6 +59,7 @@ jobs:
execute:
name: "${{ inputs.key }}"
runs-on: ubuntu-latest
+ if: ${{ inputs.execute }}
steps:
- name: Checkout repository
uses: actions/checkout@v4
@@ -84,6 +90,22 @@ jobs:
run: ${{ inputs.script }}
timeout-minutes: 60
+ - name: Publish Test Report
+ if: ${{ hashFiles('**/TEST-*.xml') != '' }}
+ uses: mikepenz/action-junit-report@v5
+ env:
+ NODE_OPTIONS: "--max_old_space_size=4096"
+ with:
+ check_name: "Test Report - ${{ inputs.key }}"
+ report_paths: '**/TEST-*.xml'
+ detailed_summary: true
+ flaky_summary: true
+ annotate_only: true
+ fail_on_failure: true
+ check_retries: true
+ truncate_stack_traces: false
+ job_summary: false
+
- name: Upload artifact
uses: actions/upload-artifact@v4
if: ${{ failure() || inputs.artifact_prefix != '' }}
diff --git a/check_test_suite.py b/check_test_suite.py
deleted file mode 100755
index 62a5902421a..00000000000
--- a/check_test_suite.py
+++ /dev/null
@@ -1,192 +0,0 @@
-#!/usr/bin/env python3
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import subprocess
-import sys
-
-# this script does some primitive examination of git diff to determine if a
test suite needs to be run or not
-
-# these jobs should always be run, no matter what
-always_run_jobs = ['license checks', '(openjdk17) packaging check']
-
-# ignore changes to these files completely since they don't impact CI, if the
changes are only to these files then all
-# of CI can be skipped. however, jobs which are always run will still be run
even if only these files are changed
-ignore_prefixes = ['.github', '.idea', '.asf.yaml', '.backportrc.json',
'.codecov.yml', '.dockerignore', '.gitignore',
- '.lgtm.yml', 'CONTRIBUTING.md', 'setup-hooks.sh',
'upload.sh', 'dev', 'distribution/docker',
- 'distribution/asf-release-process-guide.md',
- 'owasp-dependency-check-suppressions.xml', 'licenses']
-
-script_prefixes = ['check_test_suite.py', 'check_test_suite_test.py']
-script_job = ['script checks']
-
-# these files are docs changes
-# if changes are limited to this set then we can skip web-console and java
-# if no changes in this set we can skip docs
-docs_prefixes = ['docs/', 'website/']
-# travis docs job name
-docs_jobs = ['docs']
-# these files are web-console changes
-# if changes are limited to this set then we can skip docs and java
-# if no changes in this set we can skip web-console
-web_console_prefixes = ['web-console/']
-# travis web-console job name
-web_console_jobs = ['web console', 'web console end-to-end test']
-web_console_still_run_for_java_jobs = ['web console end-to-end test']
-
-
-def check_ignore(file):
- is_always_ignore = True in (file.startswith(prefix) for prefix in
ignore_prefixes)
- if is_always_ignore:
- print("found ignorable file change: {}".format(file))
- return is_always_ignore
-
-def check_testable_script(file):
- is_script = True in (file.startswith(prefix) for prefix in script_prefixes)
- if is_script:
- print("found script file change: {}".format(file))
- return is_script
-
-def check_docs(file):
- is_docs = True in (file.startswith(prefix) for prefix in docs_prefixes)
- if is_docs:
- print("found docs file change: {}".format(file))
- return is_docs
-
-
-def check_console(file):
- is_console = True in (file.startswith(prefix) for prefix in
web_console_prefixes)
- if is_console:
- print("found web-console file change: {}".format(file))
- return is_console
-
-
-def check_should_run_suite(suite, diff_files):
- """
- try to determine if a test suite should run or not given a set files in a
diff
-
- :param suite: travis job name
- :param diff_files: files changed in git diff
- :return: True if the supplied suite needs to run, False otherwise
- """
-
- if suite in always_run_jobs:
- # you gotta do what you gotta do
- return True
-
- all_ignore = True
- any_docs = False
- all_docs = True
- any_console = False
- all_console = True
- any_java = False
- any_testable_script = False
- all_testable_script = True
-
- # go over all of the files in the diff and collect some information about
the diff contents, we'll use this later
- # to decide whether or not to run the suite
- for f in diff_files:
- is_ignore = check_ignore(f)
- all_ignore = all_ignore and is_ignore
- is_docs = check_docs(f)
- any_docs = any_docs or is_docs
- all_docs = all_docs and is_docs
- is_console = check_console(f)
- any_console = any_console or is_console
- all_console = all_console and is_console
- is_script = check_testable_script(f)
- any_testable_script = any_testable_script or is_script
- all_testable_script = all_testable_script and is_script
- any_java = any_java or (not is_ignore and not is_docs and not
is_console and not is_script)
-
- # if everything is ignorable, we can skip this suite
- if all_ignore:
- return False
- # if the test suite is a doc job, return true if any of the files changed
were docs
- if suite in docs_jobs:
- return any_docs
- # if all of the changes are docs paths, but the current suite is not a
docs job, we can skip
- if all_docs:
- return False
- if suite in web_console_still_run_for_java_jobs:
- return any_console or any_java
- # if the test suite is a web console job, return true if any of the
changes are web console files
- if suite in web_console_jobs:
- return any_console
- # if all of the changes are web console paths, but the current suite is
not a web console job, we can skip
- if all_console:
- return False
- if suite in script_job:
- return any_testable_script
- if all_testable_script:
- return False
-
- # if all of the files belong to known non-java groups, we can also skip
java
- # note that this should probably be reworked to much more selectively run
the java jobs depending on the diff
- if not any_java:
- return False
-
- # we don't know we can skip for sure, so lets run it
- return True
-
-
-def failWithUsage():
- sys.stderr.write("usage: check_test_suite.py <test-suite-name>\n")
- sys.stderr.write(" e.g., check_test_suite.py docs")
- sys.exit(1)
-
-
-if __name__ == '__main__':
- suite_name = ""
-
- # when run by travis, we run this script without arguments, so collect the
test suite name from environment
- # variables. if it doesn't exist, fail
- if len(sys.argv) == 1:
- if 'TRAVIS_JOB_NAME' in os.environ:
- suite_name = os.environ['TRAVIS_JOB_NAME']
- else:
- failWithUsage()
- elif len(sys.argv) == 2:
- # to help with testing, can explicitly pass a test suite name
- suite_name = sys.argv[1]
- else:
- failWithUsage()
-
- # we only selectively run CI for PR builds, branch builds such as master
and releases will always run all suites
- is_pr = False
- if 'TRAVIS_PULL_REQUEST' in os.environ and
os.environ['TRAVIS_PULL_REQUEST'] != 'false':
- is_pr = True
-
- if not is_pr:
- print("Not a pull request build, need to run all test suites")
- sys.exit(1)
-
- # this looks like it only gets the last commit, but the way travis PR
builds work this actually gets the complete
- # diff (since all commits from the PR are rebased onto the target branch
and added as a single commit)
- all_changed_files_string = subprocess.check_output("git diff --name-only
HEAD~1", shell=True).decode('UTF-8')
- all_changed_files = all_changed_files_string.splitlines()
- print("Checking if suite '{}' needs to run test on
diff:\n{}".format(suite_name, all_changed_files_string))
-
- # we should always run all test suites for builds that are not for a pull
request
- needs_run = check_should_run_suite(suite_name, all_changed_files)
-
- if needs_run:
- print("Changes detected, need to run test suite
'{}'".format(suite_name))
- sys.exit(1)
-
- print("No applicable changes detected, can skip test suite
'{}'".format(suite_name))
- sys.exit(0)
diff --git a/check_test_suite_test.py b/check_test_suite_test.py
deleted file mode 100755
index 7970e449df7..00000000000
--- a/check_test_suite_test.py
+++ /dev/null
@@ -1,154 +0,0 @@
-#!/usr/bin/env python3
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import unittest
-import check_test_suite
-
-class CheckTestSuite(unittest.TestCase):
- def test_always_run(self):
- for job in check_test_suite.always_run_jobs:
- self.assertEqual(True,
check_test_suite.check_should_run_suite(job, ['docs/ingestion/index.md']))
- self.assertEqual(True,
check_test_suite.check_should_run_suite(job,
['web-console/src/views/index.ts']))
- self.assertEqual(True,
check_test_suite.check_should_run_suite(job,
['core/src/main/java/org/apache/druid/math/expr/Expr.java']))
-
- def test_docs(self):
- self.assertEqual(False,
check_test_suite.check_docs('check_test_suite_test.py'))
- self.assertEqual(True,
check_test_suite.check_docs('website/core/Footer.js'))
- self.assertEqual(True,
check_test_suite.check_docs('docs/ingestion/index.md'))
-
- for job in check_test_suite.docs_jobs:
- self.assertEqual(
- True,
- check_test_suite.check_should_run_suite(job,
['check_test_suite_test.py', 'docs/ingestion/index.md'])
- )
- self.assertEqual(
- False,
- check_test_suite.check_should_run_suite(
- job,
- ['check_test_suite_test.py',
'core/src/main/java/org/apache/druid/math/expr/Expr.java']
- )
- )
-
- def test_web_console(self):
- web_console_job = 'web console'
- e2e_job = 'web console end-to-end test'
- self.assertEqual(False,
check_test_suite.check_console('check_test_suite_test.py'))
- self.assertEqual(False,
check_test_suite.check_console('website/core/Footer.js'))
- self.assertEqual(True,
check_test_suite.check_console('web-console/assets/azure.png'))
- self.assertEqual(True,
check_test_suite.check_console('web-console/src/views/index.ts'))
- self.assertEqual(True,
check_test_suite.check_console('web-console/unified-console.html'))
-
- self.assertEqual(
- True,
- check_test_suite.check_should_run_suite(
- web_console_job,
- ['check_test_suite_test.py',
'web-console/unified-console.html']
- )
- )
- self.assertEqual(
- False,
- check_test_suite.check_should_run_suite(
- web_console_job,
- ['check_test_suite_test.py',
'core/src/main/java/org/apache/druid/math/expr/Expr.java']
- )
- )
- self.assertEqual(
- True,
- check_test_suite.check_should_run_suite(
- e2e_job,
- ['check_test_suite_test.py',
'web-console/unified-console.html']
- )
- )
- self.assertEqual(
- True,
- check_test_suite.check_should_run_suite(
- e2e_job,
- ['check_test_suite_test.py',
'core/src/main/java/org/apache/druid/math/expr/Expr.java']
- )
- )
-
- def test_testable_script(self):
- self.assertEqual(True,
check_test_suite.check_testable_script('check_test_suite.py'))
- self.assertEqual(True,
check_test_suite.check_testable_script('check_test_suite_test.py'))
-
- script_job = 'script checks'
- some_java_job = 'spotbugs checks'
- self.assertEqual(
- False,
- check_test_suite.check_should_run_suite(
- script_job,
- ['core/src/main/java/org/apache/druid/math/expr/Expr.java']
- )
- )
- self.assertEqual(
- True,
- check_test_suite.check_should_run_suite(
- some_java_job,
- ['check_test_suite_test.py',
'core/src/main/java/org/apache/druid/math/expr/Expr.java']
- )
- )
- self.assertEqual(
- True,
- check_test_suite.check_should_run_suite(
- some_java_job,
- ['check_test_suite_test.py',
'core/src/main/java/org/apache/druid/math/expr/Expr.java']
- )
- )
- self.assertEqual(
- False,
- check_test_suite.check_should_run_suite(
- some_java_job,
- ['check_test_suite_test.py']
- )
- )
-
- def test_some_java(self):
-
- some_java_job = 'spotbugs checks'
- some_non_java_diffs = [
- ['check_test_suite_test.py'],
- ['website/core/Footer.js'],
- ['web-console/src/views/index.ts'],
- ['licenses/foo.mit'],
- ['check_test_suite_test.py', 'website/core/Footer.js',
'web-console/unified-console.html', 'owasp-dependency-check-suppressions.xml']
- ]
- some_java_diffs = [
- ['core/src/main/java/org/apache/druid/math/expr/Expr.java'],
-
['processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlan.java'],
- ['check_test_suite_test.py', 'website/core/Footer.js',
'web-console/unified-console.html',
'core/src/main/java/org/apache/druid/math/expr/Expr.java']
- ]
-
- for false_diff in some_non_java_diffs:
- self.assertEqual(
- False,
- check_test_suite.check_should_run_suite(
- some_java_job,
- false_diff
- )
- )
- for true_diff in some_java_diffs:
- self.assertEqual(
- True,
- check_test_suite.check_should_run_suite(
- some_java_job,
- true_diff
- )
- )
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/licenses.yaml b/licenses.yaml
index d549eb5f596..83cd8176f35 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -623,7 +623,7 @@ name: Apache Commons Compress
license_category: binary
module: java-core
license_name: Apache License version 2.0
-version: 1.26.0
+version: 1.27.0
libraries:
- org.apache.commons: commons-compress
notices:
diff --git a/pom.xml b/pom.xml
index 57d2defe0d6..8e8b988c507 100644
--- a/pom.xml
+++ b/pom.xml
@@ -608,7 +608,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
- <version>1.26.0</version>
+ <version>1.27.0</version>
</dependency>
<dependency>
<groupId>org.tukaani</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
