This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 63ddae805a6 ci: maven cache diet; static checks re-homed; pre-pull and
retries purged (#20734)
63ddae805a6 is described below
commit 63ddae805a6a386ecab857c43869d59e46098e50
Author: Zoltan Haindrich <[email protected]>
AuthorDate: Thu Oct 29 08:24:56 2026 +0100
ci: maven cache diet; static checks re-homed; pre-pull and retries purged
(#20734)
---
.github/scripts/dir_manifest | 64 ++++++++++++++++++++++++++
.github/scripts/openrewrite.sh | 2 +-
.github/scripts/packaging-check.sh | 21 +++------
.github/scripts/static-checks-maven.sh | 14 +-----
.github/scripts/strict-compilation.sh | 2 +-
.github/workflows/ci.yml | 19 ++++++++
.github/workflows/docker-tests.yml | 7 +--
.github/workflows/static-checks.yml | 19 --------
.github/workflows/worker.yml | 70 ++++++++++-------------------
.mvn/maven.config | 1 +
AGENTS.md | 10 ++---
embedded-tests/pom.xml | 6 ---
extensions-core/avro-extensions/pom.xml | 7 ---
extensions-core/protobuf-extensions/pom.xml | 7 ---
pom.xml | 8 ++--
quidem-ut/pom.xml | 4 --
16 files changed, 129 insertions(+), 132 deletions(-)
diff --git a/.github/scripts/dir_manifest b/.github/scripts/dir_manifest
new file mode 100755
index 00000000000..35f79f22d93
--- /dev/null
+++ b/.github/scripts/dir_manifest
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Writes a "<path>\t<size>" manifest of every file under <dir> to <out>; a
missing <dir> yields an empty manifest.
+# Given <prev>, an earlier manifest, also reports files added and removed
since; a file whose size changed counts as both.
+# With -a <marker>, also writes <out> with suffix -dormant.txt: files not
accessed (atime) since <marker> was modified.
+#
+# Usage: dir_manifest [-a <marker>] <dir> <out> [<prev>]
+
+set -eu
+
+MARKER=
+if [ "${1:-}" = "-a" ]; then
+ MARKER=$2
+ shift 2
+fi
+
+DIR=$1
+OUT=$2
+PREV=${3:-}
+
+export LC_ALL=C
+
+list() {
+ if [ -d "$DIR" ]; then
+ find "$DIR" -type f "$@" -printf '%P\t%s\n' | sort
+ fi
+}
+
+mkdir -p "$(dirname "$OUT")"
+list > "$OUT"
+
+summarize() {
+ awk -F'\t' -v label="$1" '{ n++; s += $2 } END { printf "%s %8d files ฮฃ
%10.1f MB\n", label, n, s / 1048576 }'
+}
+
+if [ -n "$PREV" ]; then
+ summarize "๐ before " < "$PREV"
+ summarize "๐ after " < "$OUT"
+ comm -13 "$PREV" "$OUT" | summarize "โ added "
+ comm -23 "$PREV" "$OUT" | summarize "โ removed"
+else
+ summarize "๐ total " < "$OUT"
+fi
+
+if [ -n "$MARKER" ]; then
+ DORMANT=${OUT%.txt}-dormant.txt
+ comm -23 "$OUT" <(list -anewer "$MARKER") > "$DORMANT"
+ summarize "๐ค dormant" < "$DORMANT"
+fi
diff --git a/.github/scripts/openrewrite.sh b/.github/scripts/openrewrite.sh
index c0ab967ea55..cefd452ab66 100755
--- a/.github/scripts/openrewrite.sh
+++ b/.github/scripts/openrewrite.sh
@@ -21,6 +21,6 @@ set -x
export MAVEN_OPTS=-Xmx8g
echo 'Running Maven install...'
-./mvnw -B clean install -q -ff -pl '!distribution' -P skip-tests
-Dweb.console.skip=true -T1C
+./mvnw -B install -q -ff -pl '!distribution' -P skip-tests
-Dweb.console.skip=true -Dcyclonedx.skip
./mvnw -B rewrite:dryRun -Dweb.console.skip=true
diff --git a/.github/scripts/packaging-check.sh
b/.github/scripts/packaging-check.sh
index 2f5204aafc0..34584f57c8b 100755
--- a/.github/scripts/packaging-check.sh
+++ b/.github/scripts/packaging-check.sh
@@ -19,17 +19,10 @@ set -e
set -x
./.github/scripts/setup_generate_license.sh
-# This job is the single place in CI that validates everything a release build
-# produces: javadoc and source jars, the binary and source distribution
-# assemblies, and the license dependency reports. The apache-release profile is
-# enabled here so that the Docker test job only needs to build the binary
-# tarball. GPG signing and the OWASP dependency check are skipped as they are
not
-# meaningful in CI. RAT is not run here: apache-rat-plugin is not safe to run
in
-# a parallel (-T) build, so the single-threaded RAT pass in
static-checks-maven.sh
-# covers it instead.
-./mvnw -B clean install -Papache-release --fail-at-end \
- -pl '!benchmarks, !distribution' -P skip-tests -Dweb.console.skip=false -T1C
\
- -Dgpg.skip -Ddependency-check.skip
-./mvnw -B install -Papache-release -Pdist -Pbundle-contrib-exts --fail-at-end \
- -pl 'distribution' -P skip-tests -Dweb.console.skip=false -T1C \
- -Dgpg.skip -Ddependency-check.skip
+CODE_CHECKS_SKIP="-Dcheckstyle.skip -Dpmd.skip -Dforbiddenapis.skip
-Danimal.sniffer.skip"
+./mvnw -B install -Papache-release -Prat --fail-at-end \
+ -pl '!benchmarks, !distribution' -P skip-tests -Dweb.console.skip=false \
+ -Dgpg.skip -Ddependency-check.skip $CODE_CHECKS_SKIP
+./mvnw -B install -Papache-release -Prat -Pdist -Pbundle-contrib-exts
--fail-at-end \
+ -pl 'distribution' -P skip-tests -Dweb.console.skip=false \
+ -Dgpg.skip -Ddependency-check.skip $CODE_CHECKS_SKIP
diff --git a/.github/scripts/static-checks-maven.sh
b/.github/scripts/static-checks-maven.sh
index b50e03dedef..fdb5f2524a4 100755
--- a/.github/scripts/static-checks-maven.sh
+++ b/.github/scripts/static-checks-maven.sh
@@ -19,21 +19,11 @@ set -e
set -x
echo 'Running Maven install...'
-./mvnw -B clean install -q -ff -pl '!distribution' -P skip-tests
-Dweb.console.skip=true -Dmaven.javadoc.skip=true -T1C
+./mvnw -B install -q -ff -pl '!distribution' -P skip-tests
-Dweb.console.skip=true -Dmaven.javadoc.skip=true -Dcyclonedx.skip
./mvnw -B checkstyle:checkstyle --fail-at-end
-# Repo-wide RAT check. This is the only RAT pass in CI. It must run without -T:
-# apache-rat-plugin 0.17+ shares per-run state across modules, so parallel runs
-# can drop configured excludes or throw NPEs (RAT-553, fixed upstream by
RAT-573).
-./mvnw -B apache-rat:check -Prat --fail-at-end \
-
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
\
- -Drat.consoleOutput=true
-
-# The license dependency reports and check-licenses.py that used to run via
-# license_checks_script.sh are covered by the packaging-check job, which builds
-# the distribution with the apache-release profile. Not repeated here.
-# ./.github/scripts/license_checks_script.sh
+./mvnw -B apache-rat:check -Prat --fail-at-end -Drat.consoleOutput=true
./.github/scripts/analyze_dependencies_script.sh
diff --git a/.github/scripts/strict-compilation.sh
b/.github/scripts/strict-compilation.sh
index ab6cc40420c..05ad6c5e09f 100755
--- a/.github/scripts/strict-compilation.sh
+++ b/.github/scripts/strict-compilation.sh
@@ -18,4 +18,4 @@
set -e
set -x
-./mvnw -B clean -DstrictCompile test-compile --fail-at-end -P skip-tests
-Dweb.console.skip=true -T1C
+./mvnw -B -DstrictCompile test-compile --fail-at-end -P skip-tests
-Dweb.console.skip=true
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 50f9eba512d..d3f8b8b5d72 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -62,3 +62,22 @@ jobs:
script: .github/scripts/run_unit-tests ${{ matrix.args }}
-Dmaven.test.failure.ignore=true
key: ${{ matrix.name }}
artifact_prefix: "unit-test-reports-jdk25"
+
+ static-checks-maven:
+ uses: ./.github/workflows/worker.yml
+ with:
+ script: .github/scripts/static-checks-maven.sh
+ key: "static-checks-maven"
+
+ strict-compilation:
+ uses: ./.github/workflows/worker.yml
+ with:
+ script: .github/scripts/strict-compilation.sh
+ key: "strict-compilation"
+
+ openrewrite:
+ uses: ./.github/workflows/worker.yml
+ with:
+ script: .github/scripts/openrewrite.sh
+ key: "openrewrite"
+ artifact_prefix: "openrewrite-patch"
diff --git a/.github/workflows/docker-tests.yml
b/.github/workflows/docker-tests.yml
index 0da464c489c..42dc213405b 100644
--- a/.github/workflows/docker-tests.yml
+++ b/.github/workflows/docker-tests.yml
@@ -33,17 +33,14 @@ jobs:
cache: 'maven'
- name: Build the Druid distribution for the Docker image
# Build only what the Docker image needs: the binary distribution
tarball.
- # Release-only work (javadoc and source jars, source-release assembly,
- # license dependency reports) and the static checks are validated by
the
- # Static Checks CI workflow (packaging-check and static-checks-maven),
so
- # they are skipped here. Same flags as the builder stage in
+ # Release-only work and static checks are skipped. Same flags as the
builder stage in
# distribution/docker/Dockerfile. This also installs every reactor
module
# into the local Maven repository, which run_docker-tests relies on.
# benchmarks is excluded: nothing in the distribution or the Docker
tests
# depends on it. The web console assets are skipped too: no Docker test
# uses the console, the router tolerates the missing assets, and the
# console is fully built and packaged by packaging-check on every
commit.
- run: ./mvnw -B -T1C clean install -pl '!benchmarks'
-Pdist,bundle-contrib-exts,skip-static-checks,skip-tests -Dweb.console.skip=true
+ run: ./mvnw -B clean install -pl '!benchmarks'
-Pdist,bundle-contrib-exts,skip-static-checks,skip-tests -Dweb.console.skip=true
- name: Build the Docker image
run: |
set -o pipefail
diff --git a/.github/workflows/static-checks.yml
b/.github/workflows/static-checks.yml
index c8dddd98a0b..11dcac39a38 100644
--- a/.github/workflows/static-checks.yml
+++ b/.github/workflows/static-checks.yml
@@ -42,25 +42,6 @@ jobs:
jdk: ${{ matrix.jdk }}
key: "packaging-check-jdk${{ matrix.jdk }}"
- static-checks-maven:
- uses: ./.github/workflows/worker.yml
- with:
- script: .github/scripts/static-checks-maven.sh
- key: "static-checks-maven"
-
- strict-compilation:
- uses: ./.github/workflows/worker.yml
- with:
- script: .github/scripts/strict-compilation.sh
- key: "strict-compilation"
-
- openrewrite:
- uses: ./.github/workflows/worker.yml
- with:
- script: .github/scripts/openrewrite.sh
- key: "openrewrite"
- artifact_prefix: "openrewrite-patch"
-
web-checks:
uses: ./.github/workflows/worker.yml
with:
diff --git a/.github/workflows/worker.yml b/.github/workflows/worker.yml
index 31e55df23ff..795f271186c 100644
--- a/.github/workflows/worker.yml
+++ b/.github/workflows/worker.yml
@@ -61,78 +61,42 @@ jobs:
runs-on: ubuntu-24.04
if: ${{ inputs.execute }}
steps:
- - name: Checkout repository
+ - name: ๐ฅ checkout
uses: actions/checkout@v7
with:
fetch-depth: 500
- - uses: actions/setup-java@v6
+ - name: โ JDK ${{ inputs.jdk }} + .m2 cache
+ uses: actions/setup-java@v6
with:
distribution: 'zulu'
java-version: ${{ inputs.jdk }}
cache: 'maven'
- # Keep parent-POM resolution separate and before the worker script so
transient
- # Maven Central failures are retried before job-specific build or test
work starts.
- - name: Resolve Apache parent POM
+ - name: ๐ธ .m2 snapshot
run: |
- set -eu
- for attempt in 1 2 3; do
- if ./mvnw -B -U -N validate -Pskip-static-checks
-Dweb.console.skip=true; then
- exit 0
- fi
- if [ "$attempt" -eq 3 ]; then
- exit 1
- fi
- echo "::warning ::Maven parent resolution failed (attempt
$attempt); retrying in 10s"
- sleep 10
- done
+ ./.github/scripts/dir_manifest ~/.m2/repository
target/ci-debug/m2-before.txt
+ touch "$RUNNER_TEMP/m2-marker"
- - name: Download required artifacts
+ - name: ๐ฆ fetch artifacts
if: ${{ inputs.artifacts_to_download != '' }}
uses: actions/download-artifact@v8
with:
pattern: ${{ inputs.artifacts_to_download }}
merge-multiple: true
- - name: Prepare GITHUB_ENV
+ - name: โ๏ธ env + JVM diagnostics
run: |
set -x
./.github/scripts/setup_test_profiling_env.sh ${{ inputs.jdk }}
run_id=$GITHUB_RUN_ID run_number=$GITHUB_RUN_NUMBER
run_attempt=$GITHUB_RUN_ATTEMPT key=${{ inputs.key }}
event_ref=$GITHUB_EVENT_REF run_url=$GITHUB_RUN_URL >> $GITHUB_ENV
echo "HASH=$(echo -n "${{ inputs.key }}" | sha256sum | cut -c-8)" >>
$GITHUB_ENV
./.github/scripts/collect_jvm_diagnostics &
- - name: Pre-pull Testcontainers images for unit tests
- if: ${{ contains(inputs.script, 'run_unit-tests') }}
- run: |
- set -eu
- images=(
- apache/kafka:4.2.0
- postgres:16-alpine
- rustfs/rustfs:1.0.0
- hashicorp/consul:1.18
- tabulario/iceberg-rest:1.6.0
- localstack/localstack:4.13.1
- mcr.microsoft.com/azure-storage/azurite:3.35.0
- osixia/openldap:1.5.0
- rancher/k3s:v1.35.0-k3s1
- )
- for image in "${images[@]}"; do
- for attempt in 1 2 3; do
- if docker pull --quiet "$image"; then
- break
- fi
- echo "::warning ::docker pull $image failed (attempt $attempt);
retrying in 10s"
- sleep 10
- done
- done
- timeout-minutes: 10
-
- - name: 'Execute: ${{ inputs.script }}'
+ - name: 'โก ${{ inputs.script }}'
run: ${{ inputs.script }}
timeout-minutes: 60
- - name: Publish Test Report
+ - name: ๐งพ test report
if: ${{ hashFiles('**/TEST-*.xml') != '' }}
uses: mikepenz/action-junit-report@v6
env:
@@ -148,7 +112,19 @@ jobs:
truncate_stack_traces: false
job_summary: false
- - name: Upload artifact
+ - name: ๐ .m2 diff โ build
+ if: ${{ always() }}
+ run: ./.github/scripts/dir_manifest -a "$RUNNER_TEMP/m2-marker"
~/.m2/repository target/ci-debug/m2-build.txt target/ci-debug/m2-before.txt
+
+ - name: ๐๏ธ .m2 purge org/apache/druid
+ if: ${{ always() }}
+ run: rm -rf ~/.m2/repository/org/apache/druid
+
+ - name: ๐ .m2 diff โ cache
+ if: ${{ always() }}
+ run: ./.github/scripts/dir_manifest ~/.m2/repository
target/ci-debug/m2-after.txt target/ci-debug/m2-before.txt
+
+ - name: ๐ค upload artifact
uses: actions/upload-artifact@v7
if: ${{ failure() || inputs.artifact_prefix != '' }}
with:
diff --git a/.mvn/maven.config b/.mvn/maven.config
new file mode 100644
index 00000000000..698df0fd6e5
--- /dev/null
+++ b/.mvn/maven.config
@@ -0,0 +1 @@
+-Daether.remoteRepositoryFilter.prefixes=false
diff --git a/AGENTS.md b/AGENTS.md
index 1b18a79fe0f..5a24e570e26 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -69,16 +69,16 @@ When adding or migrating tests, reuse these existing
test-scope helpers:
## Running Tests
-Use these flags for faster tests: `-Pskip-static-checks
-Dweb.console.skip=true -T1C`
+Use these flags for faster tests: `-Pskip-static-checks
-Dweb.console.skip=true`
**Single test method:**
```
-mvn test -pl sql -am
-Dtest="org.apache.druid.sql.calcite.CalciteQueryTest#testFoo"
-Dsurefire.failIfNoSpecifiedTests=false -Pskip-static-checks
-Dweb.console.skip=true -T1C
+mvn test -pl sql -am
-Dtest="org.apache.druid.sql.calcite.CalciteQueryTest#testFoo"
-Dsurefire.failIfNoSpecifiedTests=false -Pskip-static-checks
-Dweb.console.skip=true
```
**All tests in a package:**
```
-mvn test -pl sql -am -Dtest="org.apache.druid.sql.**"
-Dsurefire.failIfNoSpecifiedTests=false -Pskip-static-checks
-Dweb.console.skip=true -T1C
+mvn test -pl sql -am -Dtest="org.apache.druid.sql.**"
-Dsurefire.failIfNoSpecifiedTests=false -Pskip-static-checks
-Dweb.console.skip=true
```
**Quidem (.iq) tests:**
@@ -86,13 +86,13 @@ mvn test -pl sql -am -Dtest="org.apache.druid.sql.**"
-Dsurefire.failIfNoSpecifi
To run `sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/numMerge.iq`:
```
-mvn test -pl sql -am -Dtest="org.apache.druid.quidem.SqlQuidemTest"
-Dquidem.filter=numMerge -Dsurefire.failIfNoSpecifiedTests=false
-Pskip-static-checks -Dweb.console.skip=true -T1C
+mvn test -pl sql -am -Dtest="org.apache.druid.quidem.SqlQuidemTest"
-Dquidem.filter=numMerge -Dsurefire.failIfNoSpecifiedTests=false
-Pskip-static-checks -Dweb.console.skip=true
```
To run
`quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/qaWin/basics_group_by.all.iq`:
```
-mvn test -pl quidem-ut -am -Dtest="org.apache.druid.quidem.QTest"
-Dquidem.filter=qaWin/basics_group_by.all
-Dsurefire.failIfNoSpecifiedTests=false -Pskip-static-checks
-Dweb.console.skip=true -T1C
+mvn test -pl quidem-ut -am -Dtest="org.apache.druid.quidem.QTest"
-Dquidem.filter=qaWin/basics_group_by.all
-Dsurefire.failIfNoSpecifiedTests=false -Pskip-static-checks
-Dweb.console.skip=true
```
Always use `-Dquidem.filter` to avoid running the full suite. To run multiple
tests, use commas like
diff --git a/embedded-tests/pom.xml b/embedded-tests/pom.xml
index 620b187e3ab..e502b829bb9 100644
--- a/embedded-tests/pom.xml
+++ b/embedded-tests/pom.xml
@@ -806,12 +806,6 @@
<artifactId>aws-core</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>software.amazon.awssdk</groupId>
- <artifactId>bundle</artifactId>
- <version>${aws.sdk.v2.version}</version>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-column</artifactId>
diff --git a/extensions-core/avro-extensions/pom.xml
b/extensions-core/avro-extensions/pom.xml
index 524915b5207..9e242e8266c 100644
--- a/extensions-core/avro-extensions/pom.xml
+++ b/extensions-core/avro-extensions/pom.xml
@@ -37,13 +37,6 @@
<schemarepo.version>0.1.3</schemarepo.version>
</properties>
- <repositories>
- <repository>
- <id>confluent</id>
- <url>https://packages.confluent.io/maven/</url>
- </repository>
- </repositories>
-
<dependencies>
<dependency>
<groupId>org.apache.avro</groupId>
diff --git a/extensions-core/protobuf-extensions/pom.xml
b/extensions-core/protobuf-extensions/pom.xml
index 9fdf7a7f4a3..13db487f9cc 100644
--- a/extensions-core/protobuf-extensions/pom.xml
+++ b/extensions-core/protobuf-extensions/pom.xml
@@ -38,13 +38,6 @@
<okio.version>3.18.2</okio.version>
</properties>
- <repositories>
- <repository>
- <id>confluent</id>
- <url>https://packages.confluent.io/maven/</url>
- </repository>
- </repositories>
-
<dependencyManagement>
<dependencies>
<!-- This is an indirect dependency of kafka-protobuf-provider
diff --git a/pom.xml b/pom.xml
index b60ffd806d9..df5656379be 100644
--- a/pom.xml
+++ b/pom.xml
@@ -279,9 +279,9 @@
<repositories>
<repository>
- <id>maven.org</id>
+ <id>central</id>
<name>Maven Central Repository</name>
- <url>https://repo.maven.apache.org/maven2/</url>
+ <url>https://repo.maven.apache.org/maven2</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
@@ -298,9 +298,9 @@
<pluginRepositories>
<pluginRepository>
- <id>maven.org</id>
+ <id>central</id>
<name>Maven Central Repository</name>
- <url>https://repo.maven.apache.org/maven2/</url>
+ <url>https://repo.maven.apache.org/maven2</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
diff --git a/quidem-ut/pom.xml b/quidem-ut/pom.xml
index d64d3cc9b6c..d82af40114d 100644
--- a/quidem-ut/pom.xml
+++ b/quidem-ut/pom.xml
@@ -34,10 +34,6 @@
</parent>
<repositories>
- <repository>
- <id>confluent</id>
- <url>https://packages.confluent.io/maven/</url>
- </repository>
<repository>
<id>datasets</id>
<url>https://raw.githubusercontent.com/kgyrtkirk/datasets/repo/</url>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
