This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new b47ae1a7045 [FLINK-40361][ci] Extract license check into a separate job
b47ae1a7045 is described below
commit b47ae1a7045665b9b4136ba6b0e8d1fdc270a335
Author: Sergey Nuyanzin <[email protected]>
AuthorDate: Mon Aug 10 19:50:53 2026 +0200
[FLINK-40361][ci] Extract license check into a separate job
---
tools/azure-pipelines/e2e-template.yml | 3 ++-
tools/azure-pipelines/jobs-template.yml | 35 ++++++++++++++++++++++++++++++++-
tools/ci/compile.sh | 26 +++++++++++++++++++++---
3 files changed, 59 insertions(+), 5 deletions(-)
diff --git a/tools/azure-pipelines/e2e-template.yml
b/tools/azure-pipelines/e2e-template.yml
index dcffa9b36a0..8b9bc1b7bf0 100644
--- a/tools/azure-pipelines/e2e-template.yml
+++ b/tools/azure-pipelines/e2e-template.yml
@@ -140,7 +140,8 @@ jobs:
sudo apt install ./libssl1.0.0_*.deb
displayName: Prepare E2E run
condition: not(eq(variables['SKIP'], '1'))
- - script: ${{parameters.environment}} PROFILE="$PROFILE -Dfast
-Pskip-webui-build" ./tools/ci/compile_ci.sh
+ # e2e only needs the built binary; MVN_LICENSE_CHECK_ARGS="" skips all QA
(convergence, license, structural) and the DEBUG shade logging, which run in
the license_check job.
+ - script: MVN_LICENSE_CHECK_ARGS="" ${{parameters.environment}}
PROFILE="$PROFILE -Dfast -Pskip-webui-build" ./tools/ci/compile_ci.sh
displayName: Build Flink
condition: not(eq(variables['SKIP'], '1'))
- script: ${{parameters.environment}} FLINK_DIR=`pwd`/build-target
./tools/ci/uploading_watchdog.sh flink-end-to-end-tests/run-nightly-tests.sh
${{parameters.group}}
diff --git a/tools/azure-pipelines/jobs-template.yml
b/tools/azure-pipelines/jobs-template.yml
index d86b5210465..16f6864838d 100644
--- a/tools/azure-pipelines/jobs-template.yml
+++ b/tools/azure-pipelines/jobs-template.yml
@@ -67,8 +67,9 @@ jobs:
echo "##vso[task.setvariable
variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH"
displayName: "Set JDK"
# Compile
+ # Fast multi-threaded build that only produces the test artifact; all QA
(structural + convergence + license) runs single-threaded in the license_check
job (MVN_LICENSE_CHECK_ARGS="" disables it here).
- script: |
- ${{parameters.environment}} ./tools/ci/compile_ci.sh || exit $?
+ MVN_LICENSE_CHECK_ARGS="" ${{parameters.environment}}
./tools/ci/compile_ci.sh -Dfast || exit $?
./tools/ci/create_build_artifact.sh
displayName: Compile
@@ -78,6 +79,38 @@ jobs:
targetPath: $(FLINK_ARTIFACT_DIR)
artifact: FlinkCompileArtifact-${{parameters.stage_name}}
+# All QA (structural + convergence + license) runs single-threaded here, off
the critical path (no dependsOn), so the compile job can build fast with -Dfast
-T1C.
+- job: license_check_${{parameters.stage_name}}
+ # succeeded() is needed to allow job cancellation
+ condition: and(succeeded(), not(eq(variables['MODE'], 'e2e')))
+ pool: ${{parameters.test_pool_definition}}
+ container: ${{parameters.container}}
+ timeoutInMinutes: 240
+ cancelTimeoutInMinutes: 1
+ workspace:
+ clean: all
+ steps:
+ # if on Azure, free up disk space
+ - script: ./tools/ci/free_disk_space.sh
+ target: host
+ condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
+ displayName: Free up disk space
+ - task: Cache@2
+ inputs:
+ key: $(PIPELINE_START_YEAR) | $(CACHE_KEY)
+ restoreKeys: $(PIPELINE_START_YEAR) | $(CACHE_FALLBACK_KEY)
+ path: $(MAVEN_CACHE_FOLDER)
+ continueOnError: true # continue the build even if the cache fails.
+ condition: not(eq('${{parameters.test_pool_definition.name}}', 'Default'))
+ displayName: Cache Maven local repo
+ - script: |
+ echo "##vso[task.setvariable
variable=JAVA_HOME]$JAVA_HOME_${{parameters.jdk}}_X64"
+ echo "##vso[task.setvariable
variable=PATH]$JAVA_HOME_${{parameters.jdk}}_X64/bin:$PATH"
+ displayName: "Set JDK"
+ # Full single-threaded build (deploy + -Pcheck-convergence) running every
structural QA check and the license check.
+ - script: ${{parameters.environment}} ./tools/ci/compile_ci.sh
+ displayName: QA checks
+
- job: test_${{parameters.stage_name}}
dependsOn: compile_${{parameters.stage_name}}
condition: and(succeeded(), not(eq(variables['MODE'], 'e2e')))
diff --git a/tools/ci/compile.sh b/tools/ci/compile.sh
index bdc53428fa6..2ca9a0afad9 100755
--- a/tools/ci/compile.sh
+++ b/tools/ci/compile.sh
@@ -58,9 +58,19 @@ echo
"==========================================================================
EXIT_CODE=0
-# run with -T1 because our maven output parsers don't support multi-threaded
builds
-$MVN clean deploy
-DaltDeploymentRepository=validation_repository::default::file:$MVN_VALIDATION_DIR
-Dflink.convergence.phase=install -Pcheck-convergence \
- -Dmaven.javadoc.skip=true -U -DskipTests
-Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=DEBUG "${@}" -T1 |
tee $MVN_CLEAN_COMPILE_OUT
+# Maven QA-check args: dependency convergence, the deploy that stages jars for
the license check, and the shade DEBUG log the bundled-optional check parses.
Default runs full QA; the fast compile job and e2e pass
MVN_LICENSE_CHECK_ARGS="" to build only (QA runs in the license_check job).
Non-empty also enables the structural QA + license blocks below.
+MVN_LICENSE_CHECK_ARGS="${MVN_LICENSE_CHECK_ARGS-deploy
-DaltDeploymentRepository=validation_repository::default::file:$MVN_VALIDATION_DIR
-Dflink.convergence.phase=install -Pcheck-convergence
-Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=DEBUG}"
+if [[ -n "$MVN_LICENSE_CHECK_ARGS" ]]; then
+ BUILD_GOAL_ARGS="$MVN_LICENSE_CHECK_ARGS"
+ MVN_COMPILE_THREADS="${MVN_COMPILE_THREADS:-1}"
+else
+ BUILD_GOAL_ARGS="install"
+ MVN_COMPILE_THREADS="${MVN_COMPILE_THREADS:-1C}"
+fi
+
+# Default -T1 because our output parsers (bundled-optional/shade) don't
support multi-threaded builds; the fast compile job sets MVN_COMPILE_THREADS=1C
together with MVN_LICENSE_CHECK_ARGS="" so those parsing checks are skipped.
+$MVN clean ${BUILD_GOAL_ARGS} \
+ -Dmaven.javadoc.skip=true -U -DskipTests "${@}" -T${MVN_COMPILE_THREADS} |
tee $MVN_CLEAN_COMPILE_OUT
EXIT_CODE=${PIPESTATUS[0]}
@@ -79,6 +89,9 @@ if [ $EXIT_CODE != 0 ]; then
exit $EXIT_CODE
fi
+# Structural QA (javadoc/scala/shaded/bundled) runs only in QA mode
(MVN_LICENSE_CHECK_ARGS non-empty); the fast compile job and e2e skip it.
+if [[ -n "$MVN_LICENSE_CHECK_ARGS" ]]; then
+
echo "============ Checking Javadocs ============"
javadoc_output=/tmp/javadoc.out
@@ -111,10 +124,17 @@ EXIT_CODE=$(($EXIT_CODE+$?))
check_shaded_artifacts_s3_fs presto
EXIT_CODE=$(($EXIT_CODE+$?))
+fi
+
+# License check needs the staged deploy dir; runs only in QA mode
(MVN_LICENSE_CHECK_ARGS non-empty).
+if [[ -n "$MVN_LICENSE_CHECK_ARGS" ]]; then
+
echo "============ Run license check ============"
find $MVN_VALIDATION_DIR
MVN=$MVN ${CI_DIR}/license_check.sh $MVN_CLEAN_COMPILE_OUT $MVN_VALIDATION_DIR
|| exit $?
+fi
+
exit $EXIT_CODE
