This is an automated email from the ASF dual-hosted git repository.
jason810496 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 3e47fc39968 Catch stale entries in the Java SDK dependency trust list
(#71869)
3e47fc39968 is described below
commit 3e47fc39968ff962954604326ff9078bfa21dea6
Author: PoAn Yang <[email protected]>
AuthorDate: Sun Sep 27 13:00:47 2026 +0800
Catch stale entries in the Java SDK dependency trust list (#71869)
---
.pre-commit-config.yaml | 19 +++
dev/breeze/doc/ci/04_selective_checks.md | 8 +-
.../src/airflow_breeze/utils/selective_checks.py | 4 +
dev/breeze/tests/test_selective_checks.py | 64 ++++----
java-sdk/README.md | 16 +-
.../regenerate_java_sdk_verification_metadata.py | 165 +++++++++++++++++++
...st_regenerate_java_sdk_verification_metadata.py | 175 +++++++++++++++++++++
7 files changed, 411 insertions(+), 40 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0633f56f91b..f7e2afa982a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -279,6 +279,25 @@ repos:
(?x)
^java-sdk/gradle\.properties$|
^java-sdk/sdk/schema/schema\.json$
+ - id: regenerate-java-sdk-verification-metadata
+ name: Regenerate the Java SDK dependency verification metadata
+ description: "Drop trusted checksums that no Java SDK build task
resolves any more"
+ entry: ./scripts/ci/prek/regenerate_java_sdk_verification_metadata.py
+ language: python
+ pass_filenames: false
+ # example/ and scala_spark_example/ are separate builds this metadata
does
+ # not cover, so their build files are excluded.
+ files: >
+ (?x)
+ ^java-sdk/build\.gradle\.kts$|
+ ^java-sdk/buildSrc/.*$|
+ ^java-sdk/gradle\.properties$|
+ ^java-sdk/gradle/libs\.versions\.toml$|
+ ^java-sdk/gradle/verification-metadata\.xml$|
+ ^java-sdk/gradle/wrapper/gradle-wrapper\.properties$|
+ ^java-sdk/settings\.gradle\.kts$|
+ ^scripts/ci/prek/regenerate_java_sdk_verification_metadata\.py$|
+
^java-sdk/(?!example/|scala_spark_example/)[^/]+/(?:build\.gradle\.kts|gradle\.properties)$
- id: update-java-sdk-readme-matrix
name: Update the Java SDK compatibility matrix in java-sdk/README.md
and Dokka module doc
entry: ./scripts/ci/prek/update_java_sdk_readme_matrix.py
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index 27e6039996f..489d60a8f73 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -519,9 +519,11 @@ when some files are not changed. Those are the rules
implemented:
type errors (see #68919)
* if no `All Python files` changed - `flynt` check is skipped
* if no `Helm files` changed - `lint-helm-chart` check is skipped
- * if no `Java SDK files` changed - `ktlint` check is skipped (it runs the
java-sdk Gradle
- wrapper, which downloads the Gradle distribution, so we avoid that
download on PRs that do
- not touch `java-sdk/`)
+ * if no `Java SDK files` changed - `ktlint` and
+ `regenerate-java-sdk-verification-metadata` checks are skipped (both run
the java-sdk
+ Gradle wrapper, which downloads the Gradle distribution, and the latter
additionally
+ resolves the whole Java SDK dependency graph from Maven Central, so we
avoid those
+ downloads on PRs that do not touch `java-sdk/`)
* if no `TS SDK files` (`ts-sdk/`) changed -
`check-ts-sdk-supervisor-schema` check is
skipped (it regenerates and diffs the generated ts-sdk file; a change to
the supervisor
wire schema alone deliberately does not trigger it - regenerating the
ts-sdk types is
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index d831e569227..f278d044bc7 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -1729,6 +1729,10 @@ class SelectiveChecks:
# on a cold cache. Skip it when no java-sdk files changed so
unrelated PRs do not
# depend on that (intermittently failing) download.
prek_hooks_to_skip.add("ktlint")
+ # Rewriting the verification metadata resolves the entire Java SDK
dependency graph
+ # from Maven Central. Skip it when no java-sdk files changed so
unrelated PRs do not
+ # depend on that resolution.
+ prek_hooks_to_skip.add("regenerate-java-sdk-verification-metadata")
if not self._matching_files(FileGroupForCi.TS_SDK_FILES,
CI_FILE_GROUP_MATCHES):
# This hook regenerates ts-sdk/src/generated/supervisor.ts from
the wire schema and
# diffs it. Schema-only changes deliberately do not trigger it:
regenerating the
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 5b76b9969cc..7f64795d061 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -114,7 +114,7 @@ ALL_SKIPPED_COMMITS_ON_NO_CI_IMAGE = (
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
- "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED = "identity,update-uv-lock"
@@ -128,7 +128,7 @@ ALL_SKIPPED_COMMITS_IF_ONLY_UI_OPENAPI_CHANGED = (
"mypy-shared-observability,mypy-shared-plugins_manager,mypy-shared-providers_discovery,"
"mypy-shared-secrets_backend,mypy-shared-secrets_masker,mypy-shared-serialization,"
"mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,"
- "mypy-task-sdk-integration-tests,update-uv-lock"
+
"mypy-task-sdk-integration-tests,regenerate-java-sdk-verification-metadata,update-uv-lock"
)
ALL_SKIPPED_COMMITS_IF_NO_UI = (
@@ -139,7 +139,7 @@ ALL_SKIPPED_COMMITS_IF_NO_UI = (
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
- "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS = (
"check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
@@ -149,7 +149,7 @@ ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS = (
"mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging,"
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
-
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,update-uv-lock"
+
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,regenerate-java-sdk-verification-metadata,update-uv-lock"
)
ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS = (
@@ -161,7 +161,7 @@ ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS = (
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
- "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
# API source/test change with NO OpenAPI spec change: the full matrix is no
longer
@@ -177,7 +177,7 @@ ALL_SKIPPED_COMMITS_IF_ONLY_API_SOURCE_CHANGED = (
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,"
"mypy-task-sdk,mypy-task-sdk-integration-tests,"
- "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_AND_UI = (
@@ -189,7 +189,7 @@ ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_AND_UI = (
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
- "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS = (
@@ -201,7 +201,7 @@ ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS = (
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
- "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
@@ -214,7 +214,7 @@ ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS = (
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
- "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS = (
@@ -225,7 +225,7 @@ ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS = (
"mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging,"
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
-
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,update-uv-lock"
+
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,regenerate-java-sdk-verification-metadata,update-uv-lock"
)
ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED = (
@@ -237,7 +237,7 @@ ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED = (
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
- "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
@@ -482,7 +482,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
-
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
),
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json": json.dumps(
@@ -529,7 +529,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,"
"mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
-
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
),
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json": json.dumps(
@@ -783,7 +783,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk-integration-tests,"
-
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
),
"skip-providers-tests": "false",
"upgrade-to-newer-dependencies": "false",
@@ -821,7 +821,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,"
-
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
),
"skip-providers-tests": "true",
"upgrade-to-newer-dependencies": "false",
@@ -884,7 +884,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
-
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
),
"skip-providers-tests": "true",
"upgrade-to-newer-dependencies": "false",
@@ -920,7 +920,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
-
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
),
"skip-providers-tests": "true",
"upgrade-to-newer-dependencies": "false",
@@ -1267,7 +1267,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
-
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
),
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json": json.dumps(
@@ -1431,7 +1431,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
-
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
),
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json": None,
@@ -1802,7 +1802,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,"
"mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
-
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
),
},
id=("Shared logging change keeps only mypy-shared-logging
among the mypy-shared-* hooks"),
@@ -1824,28 +1824,32 @@ def test_expected_output_pull_request_main(
assert_outputs_are_printed(expected_outputs, str(stderr))
[email protected]("hook", ["ktlint",
"regenerate-java-sdk-verification-metadata"])
@pytest.mark.parametrize(
- ("files", "ktlint_skipped"),
+ ("files", "hook_skipped"),
[
pytest.param(
("java-sdk/sdk/build.gradle.kts",),
False,
- id="ktlint runs when java-sdk files change",
+ id="runs when java-sdk files change",
),
pytest.param(
("SECURITY.md",),
True,
- id="ktlint skipped when no java-sdk files change",
+ id="skipped when no java-sdk files change",
),
pytest.param(
("java-sdk/README.md",),
True,
- id="ktlint skipped when only java-sdk docs change",
+ id="skipped when only java-sdk docs change",
),
],
)
-def test_ktlint_hook_only_runs_for_java_sdk_changes(files: tuple[str, ...],
ktlint_skipped: bool):
- # ktlint downloads the Gradle distribution, so it must be skipped unless
java-sdk changed.
+def test_java_sdk_gradle_hooks_only_run_for_java_sdk_changes(
+ files: tuple[str, ...], hook_skipped: bool, hook: str
+):
+ # Both hooks run the java-sdk Gradle wrapper and download from it, so they
must be skipped
+ # unless java-sdk changed.
stderr = SelectiveChecks(
files=files,
commit_ref=NEUTRAL_COMMIT,
@@ -1854,7 +1858,7 @@ def
test_ktlint_hook_only_runs_for_java_sdk_changes(files: tuple[str, ...], ktli
default_branch="main",
)
skipped_hooks =
get_outputs_from_stderr(str(stderr))["skip-prek-hooks"].split(",")
- assert ("ktlint" in skipped_hooks) is ktlint_skipped
+ assert (hook in skipped_hooks) is hook_skipped
@pytest.mark.parametrize(
@@ -2733,7 +2737,7 @@ def test_expected_output_push(
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
-
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
),
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json": json.dumps(
@@ -2774,7 +2778,7 @@ def test_expected_output_push(
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
-
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
),
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
@@ -2820,7 +2824,7 @@ def test_expected_output_push(
"mypy-shared-module_loading,mypy-shared-observability,mypy-shared-plugins_manager,"
"mypy-shared-providers_discovery,mypy-shared-secrets_backend,mypy-shared-secrets_masker,"
"mypy-shared-serialization,mypy-shared-state,mypy-shared-template_rendering,mypy-shared-timezones,mypy-task-sdk,mypy-task-sdk-integration-tests,"
-
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"regenerate-java-sdk-verification-metadata,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
),
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
diff --git a/java-sdk/README.md b/java-sdk/README.md
index 97b81903fb1..12a43cbf3c5 100644
--- a/java-sdk/README.md
+++ b/java-sdk/README.md
@@ -44,17 +44,19 @@ development tools may have further requirements (see the
toolchain in
Repositories are centralized in `settings.gradle.kts`, and dynamic and
changing versions are rejected for project dependency configurations (not for
plugin markers or detached configurations — pin those by hand). `buildSrc/`
declares its own repositories, but its dependencies *are* covered by this
metadata.
-To update a dependency or plugin, regenerate from a trusted network. The task
list must cover everything CI runs, since only what the invoked tasks resolve
gets recorded:
+To update a dependency or plugin, regenerate the file from a trusted network.
Run the command from the repository root:
```bash
-./gradlew --write-verification-metadata sha256 --refresh-dependencies \
- build \
- :sdk:dokkaGeneratePublicationHtml :sdk:dokkaGeneratePublicationJavadoc \
- sourceTarball checksumSourceTarball \
- publishToMavenLocal -PskipSigning=true
+prek run regenerate-java-sdk-verification-metadata --all-files
```
-Without `-PskipSigning=true` the signing tasks fail and Gradle still writes
metadata from the partial run. Regeneration only appends, so delete superseded
entries by hand after a version bump.
+The hook runs on its own whenever a change moves the resolved dependency set,
and it keeps failing until you stage the rewritten file too. It is a plain
script, so you can also run it directly:
+
+```bash
+uv run scripts/ci/prek/regenerate_java_sdk_verification_metadata.py
+```
+
+Gradle only appends to the file, so the script empties the component list
before regenerating. Otherwise every version bump leaves its superseded entries
behind, and they stay trusted. The script also owns the task list, which has to
cover everything CI builds, because Gradle records only what the invoked tasks
resolve.
Review every entry in the diff. Generating the file records what the
repositories served at that moment; it does not make those bytes trustworthy.
Cross-check new coordinates and checksums against the dependency's official
release information, and never bypass a failure with lenient or disabled
verification.
diff --git a/scripts/ci/prek/regenerate_java_sdk_verification_metadata.py
b/scripts/ci/prek/regenerate_java_sdk_verification_metadata.py
new file mode 100755
index 00000000000..d2148b7fb41
--- /dev/null
+++ b/scripts/ci/prek/regenerate_java_sdk_verification_metadata.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Rewrite java-sdk/gradle/verification-metadata.xml from an empty component list.
+
+Gradle only ever appends to that file, so a version bump leaves the superseded
+entries behind and they stay trusted for good. Starting from an empty list
drops
+them, which is what turns a stale checksum into a visible diff.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import subprocess
+import sys
+import time
+from collections.abc import Callable
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
+JAVA_SDK = REPO_ROOT / "java-sdk"
+METADATA = JAVA_SDK / "gradle" / "verification-metadata.xml"
+
+GRADLE_TASKS = [
+ "build",
+ ":sdk:dokkaGeneratePublicationHtml",
+ ":sdk:dokkaGeneratePublicationJavadoc",
+ "sourceTarball",
+ "checksumSourceTarball",
+ "publishToMavenLocal",
+]
+
+MAX_ATTEMPTS = 3
+RETRY_DELAY_SECONDS = 30
+
+LICENSE_HEADER = """<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->"""
+
+ADVISORY = """
+The trust list changed. Entries that disappeared are checksums nothing resolves
+any more - usually versions superseded by a dependency bump. They stayed
+trusted, so a future direct or transitive dependency could have pulled that
+exact version back in with nobody reviewing its checksum.
+
+Review every entry in the diff before staging it: generating the file records
+what the repositories served, it does not make those bytes trustworthy.
+"""
+
+
+class RegenerationFailedError(RuntimeError):
+ """Gradle could not regenerate the metadata within the attempt budget."""
+
+
+def build_empty_metadata(committed: str) -> str:
+ """Return the committed metadata with its component list emptied."""
+ kept: list[str] = []
+ for line in committed.splitlines():
+ if "<components>" in line:
+ kept += [" <components/>", "</verification-metadata>"]
+ break
+ kept.append(line)
+ return "\n".join(kept) + "\n"
+
+
+def insert_license_header(metadata: str) -> str:
+ """Put the ASF header back after the XML declaration, unless Gradle kept
one."""
+ if "Licensed to the Apache Software Foundation" in metadata:
+ return metadata
+ declaration, *rest = metadata.splitlines()
+ return "\n".join([declaration, LICENSE_HEADER, *rest]) + "\n"
+
+
+def run_gradle() -> bool:
+ """Both properties are needed for the run to reach the end: signing has no
key
+ here, and sourceTarball has no default ref."""
+ result = subprocess.run(
+ [
+ "./gradlew",
+ "--no-daemon",
+ "--write-verification-metadata",
+ "sha256",
+ "--refresh-dependencies",
+ *GRADLE_TASKS,
+ "-PskipSigning=true",
+ "-PgitRef=HEAD",
+ ],
+ cwd=JAVA_SDK,
+ check=False,
+ )
+ return result.returncode == 0
+
+
+def regenerate(
+ metadata: pathlib.Path,
+ gradle: Callable[[], bool],
+ sleep: Callable[[float], None] = time.sleep,
+) -> None:
+ """Rewrite the metadata in place, putting the committed file back if the
run never succeeds."""
+ committed = metadata.read_text()
+ restore = True
+ try:
+ for attempt in range(1, MAX_ATTEMPTS + 1):
+ print(f"==> Regenerating verification metadata (attempt
{attempt}/{MAX_ATTEMPTS})", flush=True)
+ metadata.write_text(build_empty_metadata(committed))
+ if gradle():
+
metadata.write_text(insert_license_header(metadata.read_text()))
+ restore = False
+ return
+ if attempt < MAX_ATTEMPTS:
+ print(f"Regeneration failed, retrying in
{RETRY_DELAY_SECONDS}s", file=sys.stderr, flush=True)
+ sleep(RETRY_DELAY_SECONDS)
+ raise RegenerationFailedError(f"Regeneration failed after
{MAX_ATTEMPTS} attempts")
+ finally:
+ if restore:
+ metadata.write_text(committed)
+
+
+def metadata_changed(metadata: pathlib.Path) -> bool:
+ result = subprocess.run(["git", "diff", "--quiet", "--", str(metadata)],
cwd=REPO_ROOT, check=False)
+ return result.returncode != 0
+
+
+def main() -> int:
+ try:
+ regenerate(METADATA, run_gradle)
+ except RegenerationFailedError as error:
+ print(f"ERROR: {error}", file=sys.stderr)
+ return 1
+ if metadata_changed(METADATA):
+ print(ADVISORY, file=sys.stderr)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git
a/scripts/tests/ci/prek/test_regenerate_java_sdk_verification_metadata.py
b/scripts/tests/ci/prek/test_regenerate_java_sdk_verification_metadata.py
new file mode 100644
index 00000000000..f4cc9a9c9b7
--- /dev/null
+++ b/scripts/tests/ci/prek/test_regenerate_java_sdk_verification_metadata.py
@@ -0,0 +1,175 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import pathlib
+
+import pytest
+import regenerate_java_sdk_verification_metadata as regenerator
+from regenerate_java_sdk_verification_metadata import (
+ LICENSE_HEADER,
+ METADATA,
+ RegenerationFailedError,
+ build_empty_metadata,
+ insert_license_header,
+ regenerate,
+)
+
+COMMITTED = f"""<?xml version="1.0" encoding="UTF-8"?>
+{LICENSE_HEADER}
+<verification-metadata
xmlns="https://schema.gradle.org/dependency-verification";>
+ <configuration>
+ <verify-metadata>true</verify-metadata>
+ </configuration>
+ <components>
+ <component group="org.example" name="superseded" version="1.0"/>
+ <component group="org.example" name="current" version="2.0"/>
+ </components>
+</verification-metadata>
+"""
+
+
+def component_names(metadata: str) -> list[str]:
+ return [line.split('name="')[1].split('"')[0] for line in
metadata.splitlines() if "<component " in line]
+
+
+class FakeGradle:
+ """Stands in for `gradlew --write-verification-metadata`.
+
+ Two behaviours matter: it only ever adds to the component list, and it
rewrites the
+ file without the ASF header. Dropping an entry is therefore something only
the caller
+ can arrange, by emptying the list first.
+ """
+
+ def __init__(self, metadata: pathlib.Path, failures: int = 0,
keeps_header: bool = False):
+ self.metadata = metadata
+ self.failures = failures
+ self.keeps_header = keeps_header
+ self.calls = 0
+
+ def __call__(self) -> bool:
+ self.calls += 1
+ if self.calls <= self.failures:
+ return False
+ text = self.metadata.read_text()
+ kept = [line for line in text.splitlines() if "<component " in line]
+ if not any('name="current"' in line for line in kept):
+ kept.append(' <component group="org.example" name="current"
version="2.0"/>')
+ header = [LICENSE_HEADER] if self.keeps_header else []
+ self.metadata.write_text(
+ "\n".join(
+ [
+ '<?xml version="1.0" encoding="UTF-8"?>',
+ *header,
+ '<verification-metadata
xmlns="https://schema.gradle.org/dependency-verification";>',
+ " <components>",
+ *kept,
+ " </components>",
+ "</verification-metadata>",
+ ]
+ )
+ + "\n"
+ )
+ return True
+
+
[email protected]
+def metadata(tmp_path) -> pathlib.Path:
+ path = tmp_path / "verification-metadata.xml"
+ path.write_text(COMMITTED)
+ return path
+
+
+class TestBuildEmptyMetadata:
+ def test_empties_the_component_list(self):
+ assert component_names(build_empty_metadata(COMMITTED)) == []
+
+ def test_keeps_the_header_and_the_configuration(self):
+ emptied = build_empty_metadata(COMMITTED)
+ assert LICENSE_HEADER in emptied
+ assert "<verify-metadata>true</verify-metadata>" in emptied
+
+
+class TestInsertLicenseHeader:
+ def test_inserts_the_header_after_the_xml_declaration(self):
+ stripped = '<?xml version="1.0"
encoding="UTF-8"?>\n<verification-metadata/>\n'
+ assert insert_license_header(stripped).splitlines()[1] ==
LICENSE_HEADER.splitlines()[0]
+
+ def test_leaves_an_existing_header_alone(self):
+ assert insert_license_header(COMMITTED) == COMMITTED
+
+
+class TestRegenerate:
+ def test_drops_superseded_entries_and_restores_the_header(self, metadata):
+ regenerate(metadata, FakeGradle(metadata))
+
+ assert component_names(metadata.read_text()) == ["current"]
+ assert metadata.read_text().splitlines()[1] ==
LICENSE_HEADER.splitlines()[0]
+
+ def
test_leaves_a_single_header_when_gradle_keeps_the_one_it_was_given(self,
metadata):
+ regenerate(metadata, FakeGradle(metadata, keeps_header=True))
+
+ assert metadata.read_text().count("Licensed to the Apache Software
Foundation") == 1
+
+ def test_retries_a_failing_run_and_succeeds(self, metadata):
+ gradle = FakeGradle(metadata, failures=2)
+
+ regenerate(metadata, gradle, sleep=lambda _: None)
+
+ assert gradle.calls == 3
+ assert component_names(metadata.read_text()) == ["current"]
+
+ def test_restores_the_committed_file_when_every_attempt_fails(self,
metadata):
+ gradle = FakeGradle(metadata, failures=99)
+
+ with pytest.raises(RegenerationFailedError):
+ regenerate(metadata, gradle, sleep=lambda _: None)
+
+ assert metadata.read_text() == COMMITTED
+
+
+def test_license_header_matches_the_committed_metadata():
+ committed = METADATA.read_text().splitlines()
+ start = committed.index("<!--")
+ end = committed.index("-->")
+ assert "\n".join(committed[start : end + 1]) == LICENSE_HEADER
+
+
+class TestMain:
+ def test_prints_the_advisory_when_the_metadata_changed(self, capsys,
monkeypatch):
+ monkeypatch.setattr(regenerator, "regenerate", lambda *_: None)
+ monkeypatch.setattr(regenerator, "metadata_changed", lambda _: True)
+
+ assert regenerator.main() == 0
+ assert "The trust list changed" in capsys.readouterr().err
+
+ def test_stays_quiet_when_the_metadata_is_unchanged(self, capsys,
monkeypatch):
+ monkeypatch.setattr(regenerator, "regenerate", lambda *_: None)
+ monkeypatch.setattr(regenerator, "metadata_changed", lambda _: False)
+
+ assert regenerator.main() == 0
+ assert capsys.readouterr().err == ""
+
+ def test_reports_failure_when_no_attempt_succeeds(self, capsys,
monkeypatch):
+ def never_succeeds(*_):
+ raise RegenerationFailedError("Regeneration failed after 3
attempts")
+
+ monkeypatch.setattr(regenerator, "regenerate", never_succeeds)
+
+ assert regenerator.main() == 1
+ assert "Regeneration failed after 3 attempts" in
capsys.readouterr().err
