This is an automated email from the ASF dual-hosted git repository.
zihaoxiang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 9220a97244 [Chore] Polish CI (#18088)
9220a97244 is described below
commit 9220a97244253c1223a1cb3d2c27fb5df3511497
Author: xiangzihao <[email protected]>
AuthorDate: Fri Mar 20 17:29:14 2026 +0800
[Chore] Polish CI (#18088)
---
.github/workflows/api-test.yml | 8 +-
.github/workflows/backend.yml | 10 +-
.github/workflows/docs.yml | 10 +-
.github/workflows/e2e-k8s.yml | 4 +-
.github/workflows/e2e.yml | 20 +--
.github/workflows/frontend.yml | 6 +-
.github/workflows/issue-robot.yml | 2 +-
.github/workflows/owasp-dependency-check.yaml | 4 +-
.github/workflows/publish-docker.yaml | 4 +-
.github/workflows/publish-helm-chart.yaml | 2 +-
.github/workflows/pull-request-target-robot.yml | 2 +-
.github/workflows/unit-test.yml | 188 +++++++++++++++++++++---
12 files changed, 207 insertions(+), 53 deletions(-)
diff --git a/.github/workflows/api-test.yml b/.github/workflows/api-test.yml
index b1668aad91..c9a9bca347 100644
--- a/.github/workflows/api-test.yml
+++ b/.github/workflows/api-test.yml
@@ -37,7 +37,7 @@ jobs:
outputs:
not-ignore: ${{ steps.filter.outputs.not-ignore }}
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
id: filter
with:
@@ -58,7 +58,7 @@ jobs:
sudo docker image prune --all --force
sudo docker builder prune -a
- run: df -h
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
- name: Sanity Check
@@ -120,11 +120,11 @@ jobs:
env:
RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
- name: Set up JDK 11
- uses: actions/setup-java@v4
+ uses: actions/setup-java@v5
with:
java-version: 11
distribution: 'adopt'
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 46fb46e9f3..cc78b7dc2f 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -37,7 +37,7 @@ jobs:
not-ignore: ${{ steps.filter.outputs.not-ignore }}
db-schema: ${{ steps.filter.outputs.db-schema }}
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
id: filter
with:
@@ -56,11 +56,11 @@ jobs:
java: [ '8', '11' ]
timeout-minutes: 30
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
- name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
+ uses: actions/setup-java@v5
with:
java-version: ${{ matrix.java }}
distribution: 'adopt'
@@ -114,7 +114,7 @@ jobs:
sudo docker image prune --all --force
sudo docker builder prune -a
- run: df -h
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
- name: Collect Workflow Telemetry
@@ -145,7 +145,7 @@ jobs:
- name: schema-check-with-postgresql
script: .github/workflows/schema-check/postgresql/start-job.sh
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
- name: Collect Workflow Telemetry
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index e8b24bbe4a..9ff9e94f8e 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -30,7 +30,7 @@ jobs:
timeout-minutes: 10
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Style Check
run: ./mvnw spotless:check
img-check:
@@ -40,7 +40,7 @@ jobs:
run:
working-directory: docs
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Set up Python 3.9
uses: actions/setup-python@v2
with:
@@ -54,7 +54,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
- name: Collect Workflow Telemetry
@@ -72,7 +72,7 @@ jobs:
outputs:
helm-doc: ${{ steps.filter.outputs.helm-doc }}
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
id: filter
with:
@@ -86,7 +86,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
- name: Generating helm-doc
diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml
index 47c439de97..b0b0430a58 100644
--- a/.github/workflows/e2e-k8s.yml
+++ b/.github/workflows/e2e-k8s.yml
@@ -37,7 +37,7 @@ jobs:
outputs:
not-ignore: ${{ steps.filter.outputs.not-ignore }}
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
id: filter
with:
@@ -58,7 +58,7 @@ jobs:
sudo docker image prune --all --force
sudo docker builder prune -a
- run: df -h
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
- run: df -h
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 3beaa1a53c..cc1b888a7d 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -39,7 +39,7 @@ jobs:
outputs:
not-ignore: ${{ steps.filter.outputs.not-ignore }}
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
id: filter
with:
@@ -53,7 +53,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
- name: Sanity Check
@@ -78,7 +78,7 @@ jobs:
run: |
docker save apache/dolphinscheduler-standalone-server:ci -o
/tmp/standalone-image.tar \
&& du -sh /tmp/standalone-image.tar
- - uses: actions/upload-artifact@v4
+ - uses: actions/upload-artifact@v6
name: Upload Docker Images
with:
name: standalone-image-e2e
@@ -140,15 +140,15 @@ jobs:
- run: df -h
- name: "node-cleanup"
run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
/opt/hostedtoolcache/CodeQL
- sudo docker image prune --all --force
- sudo docker builder prune -a
+ sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
/opt/hostedtoolcache/CodeQL
+ sudo docker image prune --all --force
+ sudo docker builder prune -a
- run: df -h
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
- name: Set up JDK 11
- uses: actions/setup-java@v4
+ uses: actions/setup-java@v5
with:
java-version: 11
distribution: 'adopt'
@@ -162,7 +162,7 @@ jobs:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
restore-keys: ${{ runner.os }}-maven-
- - uses: actions/download-artifact@v4
+ - uses: actions/download-artifact@v8
name: Download Docker Images
with:
name: standalone-image-e2e
@@ -176,7 +176,7 @@ jobs:
./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \
-DfailIfNoTests=false \
-Dtest=${{ matrix.case.class }} test
- - uses: actions/upload-artifact@v4
+ - uses: actions/upload-artifact@v6
if: always()
name: Upload Recording
with:
diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
index 98b4657d79..faef9a176e 100644
--- a/.github/workflows/frontend.yml
+++ b/.github/workflows/frontend.yml
@@ -42,7 +42,7 @@ jobs:
outputs:
not-ignore: ${{ steps.filter.outputs.not-ignore }}
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
id: filter
with:
@@ -59,7 +59,7 @@ jobs:
matrix:
os: [ ubuntu-latest, macos-latest ]
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
- if: matrix.os == 'ubuntu-latest'
@@ -90,7 +90,7 @@ jobs:
needs: [ build, paths-filter ]
if: always()
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Status
run: |
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{
github.event_name }} == 'pull_request' ]]; then
diff --git a/.github/workflows/issue-robot.yml
b/.github/workflows/issue-robot.yml
index 06a363d11f..01b2fb92bd 100644
--- a/.github/workflows/issue-robot.yml
+++ b/.github/workflows/issue-robot.yml
@@ -26,7 +26,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: "Checkout ${{ github.ref }}"
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
with:
persist-credentials: false
submodules: true
diff --git a/.github/workflows/owasp-dependency-check.yaml
b/.github/workflows/owasp-dependency-check.yaml
index bd347506e2..fac0002b7e 100644
--- a/.github/workflows/owasp-dependency-check.yaml
+++ b/.github/workflows/owasp-dependency-check.yaml
@@ -36,11 +36,11 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
- name: Set up JDK 11
- uses: actions/setup-java@v4
+ uses: actions/setup-java@v5
with:
java-version: 11
distribution: 'adopt'
diff --git a/.github/workflows/publish-docker.yaml
b/.github/workflows/publish-docker.yaml
index db69cfb6a9..7e7e554ffe 100644
--- a/.github/workflows/publish-docker.yaml
+++ b/.github/workflows/publish-docker.yaml
@@ -33,10 +33,10 @@ jobs:
packages: write
timeout-minutes: 30
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Cache local Maven repository
uses: actions/cache@v4
with:
diff --git a/.github/workflows/publish-helm-chart.yaml
b/.github/workflows/publish-helm-chart.yaml
index 247a55f4cc..54a522ee08 100644
--- a/.github/workflows/publish-helm-chart.yaml
+++ b/.github/workflows/publish-helm-chart.yaml
@@ -33,7 +33,7 @@ jobs:
packages: write
timeout-minutes: 30
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- name: Set environment variables
run: |
if [[ ${{ github.event_name }} == "release" ]]; then
diff --git a/.github/workflows/pull-request-target-robot.yml
b/.github/workflows/pull-request-target-robot.yml
index 938e55498a..c934e0ba1d 100644
--- a/.github/workflows/pull-request-target-robot.yml
+++ b/.github/workflows/pull-request-target-robot.yml
@@ -29,7 +29,7 @@ jobs:
timeout-minutes: 10
steps:
- name: "Checkout ${{ github.ref }}"
- uses: actions/checkout@v4
+ uses: actions/checkout@v6
with:
persist-credentials: false
submodules: true
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index 8e6c50f66e..61bbcd8a67 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -28,9 +28,6 @@ on:
- '[0-9]+.[0-9]+.[0-9]+-release'
- 'dev'
-env:
- LOG_DIR: /tmp/dolphinscheduler
-
concurrency:
group: unit-test-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
@@ -42,56 +39,190 @@ jobs:
outputs:
not-ignore: ${{ steps.filter.outputs.not-ignore }}
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
id: filter
with:
filters: |
not-ignore:
- '!(docs/**)'
+
+ generate-matrix:
+ name: Generate Module Matrix
+ runs-on: ubuntu-latest
+ outputs:
+ matrix: ${{ steps.set-matrix.outputs.matrix }}
+ steps:
+ - uses: actions/checkout@v6
+
+ - name: Parse modules from pom.xml
+ id: set-matrix
+ run: |
+ MODULES=$(python3 - <<'EOF'
+ import xml.etree.ElementTree as ET, json, re
+
+ tree = ET.parse('pom.xml')
+ ns = {'m': 'http://maven.apache.org/POM/4.0.0'}
+ root = tree.getroot()
+
+ # Support both namespaced and non-namespaced pom.xml
+ modules = root.findall('.//m:modules/m:module', ns)
+ if not modules:
+ modules = root.findall('.//modules/module')
+
+ # Exclude aggregator-only or non-testable modules
+ exclude =
re.compile(r'dolphinscheduler-bom|dolphinscheduler-dist|dolphinscheduler-microbench|dolphinscheduler-ui')
+ result = [m.text.strip() for m in modules if m.text and not
exclude.search(m.text)]
+ print(json.dumps(result))
+ EOF
+ )
+
+ echo "matrix={\"module\":$MODULES}" >> "$GITHUB_OUTPUT"
+ echo "Discovered modules: $MODULES"
+
unit-test:
- name: Unit-Test
- needs: paths-filter
+ name: Unit-Test (${{ matrix.module }} | Java ${{ matrix.java }})
+ needs: [ paths-filter, generate-matrix ]
if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') ||
(github.event_name == 'push') }}
runs-on: ubuntu-latest
strategy:
+ fail-fast: false
matrix:
- java: ['8', '11']
+ java: [ '8', '11' ]
+ module: ${{ fromJSON(needs.generate-matrix.outputs.matrix).module }}
timeout-minutes: 60
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v6
with:
submodules: true
+
- name: Collect Workflow Telemetry
uses: ./.github/actions/workflow-telemetry-action
with:
comment_on_pr: false
+
- name: Sanity Check
uses: ./.github/actions/sanity-check
with:
token: ${{ secrets.GITHUB_TOKEN }}
+
- name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
+ uses: actions/setup-java@v5
with:
java-version: ${{ matrix.java }}
distribution: 'adopt'
+
- uses: actions/cache@v4
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
restore-keys: ${{ runner.os }}-maven-
- - name: Run Unit tests
- run: export MAVEN_OPTS="-Xmx8g -XX:MetaspaceSize=256m
-XX:MaxMetaspaceSize=1024m" && ./mvnw clean verify -B -Dmaven.test.skip=false
-Dspotless.skip=true -DskipUT=false -Danalyze.skip=true
- - name: Upload coverage report to codecov
+
+ # Use -am (--also-make) so Maven automatically builds all upstream
+ # dependency modules before running tests in the target module.
+ # -DskipTests=true skips tests in dependency modules; only the
+ # target module (-pl) runs its unit tests via verify.
+ - name: Run Unit Tests (${{ matrix.module }})
+ run: |
+ export MAVEN_OPTS="-Xmx8g -XX:MetaspaceSize=256m
-XX:MaxMetaspaceSize=1024m"
+ ./mvnw verify -B \
+ -pl "${{ matrix.module }}" \
+ -am \
+ -DskipTests=true \
+ -Dmaven.test.skip=false \
+ -pl "${{ matrix.module }}" \
+ -Dspotless.skip=true \
+ -DskipUT=false \
+ -Danalyze.skip=true
+
+ - name: Collect jacoco exec files
+ if: always()
+ run: |
+ mkdir -p /tmp/jacoco-exec
+ find . -name "jacoco.exec" -exec cp --backup=numbered {}
/tmp/jacoco-exec/ \;
+
+ - name: Upload jacoco exec
+ uses: actions/upload-artifact@v6
+ if: always()
+ with:
+ name: jacoco-exec-java${{ matrix.java }}-${{ matrix.module }}
+ path: /tmp/jacoco-exec/
+ retention-days: 1
+
+ - name: Upload surefire reports
+ uses: actions/upload-artifact@v6
+ if: always()
+ with:
+ name: surefire-java${{ matrix.java }}-${{ matrix.module }}
+ path: '**/target/surefire-reports/'
+ retention-days: 7
+
+ - name: Upload coverage to Codecov
+ if: matrix.java == '11'
run: CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl
-s https://codecov.io/bash)
- # Set up JDK 17 for SonarCloud.
- - name: Set up JDK 17
- uses: actions/setup-java@v4
+ sonar:
+ name: SonarCloud Analysis
+ runs-on: ubuntu-latest
+ needs: [ paths-filter, unit-test ]
+ if: ${{ always() && (needs.paths-filter.outputs.not-ignore == 'true' ||
github.event_name == 'push') }}
+ timeout-minutes: 30
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ submodules: true
+ fetch-depth: 0
+
+ - name: Download all jacoco exec files (Java 11)
+ uses: actions/download-artifact@v8
+ with:
+ pattern: jacoco-exec-java11-*
+ path: all-jacoco-exec/
+ merge-multiple: true
+
+ - name: Set up JDK 17 for SonarCloud
+ uses: actions/setup-java@v5
with:
java-version: 17
distribution: 'adopt'
- # Sonar check need to run after jacoco report generated.
+
+ - uses: actions/cache@v4
+ with:
+ path: ~/.m2/repository
+ key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
+ restore-keys: ${{ runner.os }}-maven-
+
+ # Use jacococli.jar instead of jacoco:merge Maven goal.
+ # The 'fileSets' parameter of jacoco:merge cannot be passed via -D on
+ # the command line; it must be declared in pom.xml <configuration>.
+ # jacococli.jar merge accepts file paths directly, no XML config needed.
+ - name: Download jacococli.jar
+ run: |
+ JACOCO_VERSION=$(./mvnw help:evaluate -Dexpression=jacoco.version -q
-DforceStdout 2>/dev/null || echo "0.8.14")
+ echo "Using JaCoCo version: $JACOCO_VERSION"
+ mvn dependency:get \
+ -Dartifact=org.jacoco:org.jacoco.cli:${JACOCO_VERSION}:jar:nodeps \
+ -Ddest=/tmp/jacococli.jar \
+ -q
+ # Fallback: copy from local .m2 cache if dependency:get did not
place it at -Ddest
+ find ~/.m2 -name "org.jacoco.cli-*-nodeps.jar" | head -1 | xargs
-I{} cp {} /tmp/jacococli.jar || true
+
+ - name: Merge JaCoCo exec files via jacococli
+ run: |
+ # Collect all .exec files under the downloaded artifact directory
+ EXEC_FILES=$(find all-jacoco-exec/ -name "*.exec" | tr '\n' ' ')
+ echo "Merging exec files: $EXEC_FILES"
+ mkdir -p target
+ java -jar /tmp/jacococli.jar merge $EXEC_FILES \
+ --destfile target/merged.exec
+
+ - name: Generate merged JaCoCo XML report
+ run: |
+ ./mvnw jacoco:report \
+ -Djacoco.dataFile=target/merged.exec \
+ -Dspotless.skip=true \
+ -Dmaven.test.skip=true
+
- name: Run SonarCloud Analysis
run: >
./mvnw --batch-mode verify sonar:sonar
@@ -110,11 +241,30 @@ jobs:
-DskipUT=true
-Danalyze.skip=true
+ publish-test-results:
+ name: Publish Test Results
+ runs-on: ubuntu-latest
+ needs: unit-test
+ if: always()
+ steps:
+ - name: Download all surefire reports
+ uses: actions/download-artifact@v8
+ with:
+ pattern: surefire-java11-*
+ path: all-surefire/
+ merge-multiple: true
+
+ - name: Publish Unit Test Results
+ uses: EnricoMi/publish-unit-test-result-action@v2
+ with:
+ files: all-surefire/**/*.xml
+ comment_mode: off
+
result:
name: Unit Test
runs-on: ubuntu-latest
timeout-minutes: 5
- needs: [ unit-test, paths-filter ]
+ needs: [ unit-test, paths-filter, sonar ]
if: always()
steps:
- name: Status
@@ -127,3 +277,7 @@ jobs:
echo "Unit Test Failed!"
exit -1
fi
+ if [[ ${{ needs.sonar.result }} != 'success' ]]; then
+ echo "SonarCloud Analysis Failed!"
+ exit -1
+ fi
