uranusjr commented on code in PR #75184:
URL: https://github.com/apache/airflow/pull/75184#discussion_r4439641354
##########
scripts/ci/prek/sync_java_sdk_supervisor_schema.py:
##########
@@ -16,65 +16,110 @@
# specific language governing permissions and limitations
# under the License.
"""
-Keep the Java SDK's bundled Supervisor Schema in sync with
``airflowSupervisorSchemaVersion``.
+Refresh the Java SDK's vendored copy of the supervisor schema that the Python
Task SDK owns.
-The Gradle task ``:sdk:syncSupervisorSchema`` downloads a fresh
``schema.json`` when the
-``api_version`` inside it differs from the version declared in
``java-sdk/gradle.properties``.
-Starting Gradle for that comparison costs over a minute on a cold CI runner
(wrapper download,
-JVM start, plugin resolution), so this hook does the same comparison in Python
first and only
-hands over to Gradle when the two versions actually differ.
+``java-sdk`` vendors the schema so it builds from a source distribution with
no network, the
+way ``go-sdk`` and ``ts-sdk`` vendor theirs. It used to refresh that copy by
downloading the
+version named in ``java-sdk/gradle.properties``, which has two problems: the
remote only
+publishes released versions, so an in-progress one cannot be fetched at all,
and a schema
+change that leaves ``api_version`` alone moved nothing, so the copy went stale
unnoticed.
+
+The copy is now refreshed from the schema in this repository and compared by
content, so any
+change to the Python source moves it in the same commit. Copying is
mechanical, so this copies
+for you and fails; ``git add`` the refreshed copy and recommit.
``airflowSupervisorSchemaVersion``
+stays the declared version and is asserted against the source rather than used
to fetch it.
+
+Outside the monorepo the source is absent and the copy is already what the
distribution
+shipped, so the download is left as the fallback.
+
+Run it from the repo root, through prek:
+
+ prek run sync-java-sdk-supervisor-schema
+
+Exits 0 when the copy already matched its source, 1 when it refreshed it.
"""
from __future__ import annotations
import json
import re
+import shutil
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
-JAVA_SDK_DIR = REPO_ROOT / "java-sdk"
-GRADLE_PROPERTIES = JAVA_SDK_DIR / "gradle.properties"
-SCHEMA_FILE = JAVA_SDK_DIR / "sdk" / "schema" / "schema.json"
+
+GRADLE_PROPERTIES = Path("java-sdk/gradle.properties")
+VENDORED_SCHEMA = Path("java-sdk/sdk/schema/schema.json")
+SOURCE_SCHEMA =
Path("task-sdk/src/airflow/sdk/execution_time/schema/schema.json")
VERSION_PATTERN =
re.compile(r"^airflowSupervisorSchemaVersion\s*=\s*(\S+)\s*$", re.MULTILINE)
-def configured_version() -> str:
- match = VERSION_PATTERN.search(GRADLE_PROPERTIES.read_text())
+def configured_version(repo_root: Path = REPO_ROOT) -> str:
+ """Return the version ``gradle.properties`` declares, which the schema has
to agree with."""
+ properties = repo_root / GRADLE_PROPERTIES
+ match = VERSION_PATTERN.search(properties.read_text())
if not match:
- print(f"airflowSupervisorSchemaVersion is not set in
{GRADLE_PROPERTIES}", file=sys.stderr)
- sys.exit(1)
+ raise SystemExit(f"airflowSupervisorSchemaVersion is not set in
{GRADLE_PROPERTIES}")
return match.group(1)
-def bundled_version() -> str | None:
- if not SCHEMA_FILE.exists():
+def api_version_of(path: Path) -> str | None:
+ """Return the ``api_version`` of a schema file, or None when it is missing
or unreadable."""
+ if not path.is_file():
return None
try:
- with SCHEMA_FILE.open() as schema:
+ with path.open() as schema:
return json.load(schema).get("api_version")
except json.JSONDecodeError:
- # A truncated or conflict-marked file counts as drift; Gradle
overwrites it.
+ # A truncated or conflict-marked file counts as drift; it is
overwritten.
return None
-def main() -> int:
- expected = configured_version()
- actual = bundled_version()
- if actual == expected:
- print(f"Supervisor Schema is up-to-date (api_version={expected}).")
- return 0
- print(
- f"Supervisor Schema api_version={actual!r} differs from configured
{expected!r}, syncing via Gradle."
+def refresh_from_source(expected: str, repo_root: Path = REPO_ROOT) ->
tuple[int, str]:
Review Comment:
Requiring `airflowSupervisorSchemaVersion` to equal the source's
`api_version` makes the Java SDK's pin follow `main` every time a supervisor
version is cut. The vendored file and the pinned version are different things.
Copy the file by content, but accept any pin that exists in the bundle?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
