This is an automated email from the ASF dual-hosted git repository.
kaxil pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 71bdce5fa51 Run lang-SDK serialization conformance checks in CI
instead of prek (#75872)
71bdce5fa51 is described below
commit 71bdce5fa513b35071b9a01ffe94b81611ae4644
Author: Pierre Jeambrun <[email protected]>
AuthorDate: Wed Oct 28 22:10:43 2026 +0100
Run lang-SDK serialization conformance checks in CI instead of prek (#75872)
These checks build and run each SDK's serializer, so as prek hooks they
needed a full
Go, Java or Node toolchain and took minutes on every commit touching
Airflow's
serializer. They are tests, not static checks, so they now run as a CI test
job that
sets up only the toolchain of the SDK it checks, for the SDKs whose inputs
changed.
---
.github/workflows/ci-amd.yml | 58 ++++++++++++++
.github/workflows/ci-arm.yml | 58 ++++++++++++++
.pre-commit-config.yaml | 56 -------------
dev/breeze/doc/ci/04_selective_checks.md | 28 ++++---
.../src/airflow_breeze/utils/selective_checks.py | 52 ++++++++++---
dev/breeze/tests/test_selective_checks.py | 91 +++++++++++++++-------
go-sdk/DEVELOPMENT.md | 10 +--
go-sdk/airflow/conformance_test.go | 4 +-
java-sdk/sdk/build.gradle.kts | 2 +-
.../check_go_sdk.py} | 2 +-
.../ci/lang_sdk_serialization/check_java_sdk.py | 4 +-
.../ci/lang_sdk_serialization/check_ts_sdk.py | 7 +-
scripts/ci/lang_sdk_serialization/compare.py | 4 +-
.../test_check_go_sdk.py} | 4 +-
ts-sdk/.pre-commit-config.yaml | 16 ----
ts-sdk/tests/conformance/serialize_typescript.ts | 2 +-
16 files changed, 254 insertions(+), 144 deletions(-)
diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml
index 75c3ff8edc4..98dc1726b8f 100644
--- a/.github/workflows/ci-amd.yml
+++ b/.github/workflows/ci-amd.yml
@@ -303,6 +303,7 @@ jobs:
kubernetes-versions-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string
}}
kustomize-overlay-names: ${{
steps.selective-checks.outputs.kustomize-overlay-names }}
+ lang-sdk-conformance-sdks: ${{
steps.selective-checks.outputs.lang-sdk-conformance-sdks }}
run-kustomize-overlays-tests: ${{
steps.selective-checks.outputs.run-kustomize-overlays-tests }}
latest-versions-only: ${{
steps.selective-checks.outputs.latest-versions-only }}
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
@@ -1356,6 +1357,60 @@ jobs:
working-directory: ./java-sdk
run: ./gradlew test
+ tests-lang-sdk-conformance:
+ name: "Lang SDK serialization conformance (${{ matrix.sdk }})"
+ needs: [build-info]
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'8+16', '32+64') }}
+ timeout-minutes: 20
+ permissions:
+ contents: read
+ strategy:
+ fail-fast: false
+ matrix:
+ sdk: ${{ fromJSON(needs.build-info.outputs.lang-sdk-conformance-sdks)
}}
+ if: needs.build-info.outputs.lang-sdk-conformance-sdks != '[]'
+ steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
+ with:
+ persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
+ # Serializes the same Dags with Airflow, from airflow-core's sources, to
compare them.
+ - name: "Install uv"
+ uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 #
v10.3.0
+ with:
+ python-version: "${{ needs.build-info.outputs.default-python-version
}}"
+ - name: Setup Go
+ if: matrix.sdk == 'go'
+ uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e #
v7.0.0
+ with:
+ go-version-file: go-sdk/go.mod
+ cache-dependency-path: go-sdk/go.sum
+ - name: Setup Java
+ if: matrix.sdk == 'java'
+ uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad #
v6.1.0
+ with:
+ distribution: 'temurin'
+ java-version: ${{ env.JAVA_VERSION }}
+ - name: Setup pnpm
+ if: matrix.sdk == 'ts'
+ uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 #
v6.1.0
+ with:
+ package_json_file: ts-sdk/package.json
+ run_install: false
+ - name: Setup node
+ if: matrix.sdk == 'ts'
+ uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 #
v7.1.0
+ with:
+ node-version: "24.19.0"
+ cache: 'pnpm'
+ cache-dependency-path: ts-sdk/pnpm-lock.yaml
+ - name: "Check the ${{ matrix.sdk }} SDK serializes Dags the way Airflow
does"
+ env:
+ SDK: ${{ matrix.sdk }}
+ run: python3 "scripts/ci/lang_sdk_serialization/check_${SDK}_sdk.py"
+
build-java-sdk-docs:
name: "Java SDK docs"
needs: [build-info]
@@ -1495,6 +1550,7 @@ jobs:
- tests-go-sdk
- tests-java-sdk
- tests-ts-sdk
+ - tests-lang-sdk-conformance
- build-java-sdk-docs
- build-ts-sdk-docs
- tests-with-lowest-direct-resolution-core
@@ -1548,6 +1604,7 @@ jobs:
- tests-java-sdk
- tests-kubernetes
- tests-kustomize-overlays
+ - tests-lang-sdk-conformance
- tests-mysql-core
- tests-mysql-providers
- tests-non-db-core
@@ -1615,6 +1672,7 @@ jobs:
- tests-java-sdk
- tests-kubernetes
- tests-kustomize-overlays
+ - tests-lang-sdk-conformance
- tests-mysql-core
- tests-mysql-providers
- tests-non-db-core
diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
index 341e1e02160..8aa56eab63a 100644
--- a/.github/workflows/ci-arm.yml
+++ b/.github/workflows/ci-arm.yml
@@ -184,6 +184,7 @@ jobs:
kubernetes-versions-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string
}}
kustomize-overlay-names: ${{
steps.selective-checks.outputs.kustomize-overlay-names }}
+ lang-sdk-conformance-sdks: ${{
steps.selective-checks.outputs.lang-sdk-conformance-sdks }}
run-kustomize-overlays-tests: ${{
steps.selective-checks.outputs.run-kustomize-overlays-tests }}
latest-versions-only: ${{
steps.selective-checks.outputs.latest-versions-only }}
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
@@ -1237,6 +1238,60 @@ jobs:
working-directory: ./java-sdk
run: ./gradlew test
+ tests-lang-sdk-conformance:
+ name: "Lang SDK serialization conformance (${{ matrix.sdk }})"
+ needs: [build-info]
+ runs-on: ${{ format(fromJSON(needs.build-info.outputs.runner-type)[0],
'8+16', '32+64') }}
+ timeout-minutes: 20
+ permissions:
+ contents: read
+ strategy:
+ fail-fast: false
+ matrix:
+ sdk: ${{ fromJSON(needs.build-info.outputs.lang-sdk-conformance-sdks)
}}
+ if: needs.build-info.outputs.lang-sdk-conformance-sdks != '[]'
+ steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
+ with:
+ persist-credentials: false
+ - name: "RunsOn cost summary"
+ uses: ./.github/actions/runs-on-cost-summary
+ # Serializes the same Dags with Airflow, from airflow-core's sources, to
compare them.
+ - name: "Install uv"
+ uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 #
v10.3.0
+ with:
+ python-version: "${{ needs.build-info.outputs.default-python-version
}}"
+ - name: Setup Go
+ if: matrix.sdk == 'go'
+ uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e #
v7.0.0
+ with:
+ go-version-file: go-sdk/go.mod
+ cache-dependency-path: go-sdk/go.sum
+ - name: Setup Java
+ if: matrix.sdk == 'java'
+ uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad #
v6.1.0
+ with:
+ distribution: 'temurin'
+ java-version: ${{ env.JAVA_VERSION }}
+ - name: Setup pnpm
+ if: matrix.sdk == 'ts'
+ uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 #
v6.1.0
+ with:
+ package_json_file: ts-sdk/package.json
+ run_install: false
+ - name: Setup node
+ if: matrix.sdk == 'ts'
+ uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 #
v7.1.0
+ with:
+ node-version: "24.19.0"
+ cache: 'pnpm'
+ cache-dependency-path: ts-sdk/pnpm-lock.yaml
+ - name: "Check the ${{ matrix.sdk }} SDK serializes Dags the way Airflow
does"
+ env:
+ SDK: ${{ matrix.sdk }}
+ run: python3 "scripts/ci/lang_sdk_serialization/check_${SDK}_sdk.py"
+
build-java-sdk-docs:
name: "Java SDK docs"
needs: [build-info]
@@ -1376,6 +1431,7 @@ jobs:
- tests-go-sdk
- tests-java-sdk
- tests-ts-sdk
+ - tests-lang-sdk-conformance
- build-java-sdk-docs
- build-ts-sdk-docs
- tests-with-lowest-direct-resolution-core
@@ -1428,6 +1484,7 @@ jobs:
- tests-java-sdk
- tests-kubernetes
- tests-kustomize-overlays
+ - tests-lang-sdk-conformance
- tests-mysql-core
- tests-mysql-providers
- tests-non-db-core
@@ -1494,6 +1551,7 @@ jobs:
- tests-java-sdk
- tests-kubernetes
- tests-kustomize-overlays
+ - tests-lang-sdk-conformance
- tests-mysql-core
- tests-mysql-providers
- tests-non-db-core
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 104650a697d..77edd24df59 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -500,62 +500,6 @@ repos:
^scripts/ci/prek/check_go_sdk_generated_drift\.py$
pass_filenames: false
require_serial: true
- # check-ts-sdk-serialization-conformance in ts-sdk/ covers the SDK's own
files; this runs the same
- # check when the shared harness or Airflow's serializer changes, which
that project cannot see.
- - id: check-ts-sdk-serialization-conformance-shared
- name: Check the TS SDK serializes Dags the way Airflow does, after a
shared change
- env:
- PYTHONPYCACHEPREFIX: .build/pycache
- description: "Serialize the shared test Dags with the TS SDK and with
Airflow, and compare the two"
- entry: ./ts-sdk/scripts/ci/prek/check_serialization_conformance.py
- language: node
- additional_dependencies: ['[email protected]']
- pass_filenames: false
- require_serial: true
- files: >
- (?x)
- ^airflow-core/src/airflow/serialization/serialized_objects\.py$|
- ^airflow-core/src/airflow/serialization/schema\.json$|
- ^scripts/ci/lang_sdk_serialization/.*$
- - id: check-go-sdk-serialization-conformance
- name: Check the Go SDK serializes Dags the way Airflow does
- env:
- PYTHONPYCACHEPREFIX: .build/pycache
- description: "Serialize the shared test Dags with the Go SDK and with
Airflow, and compare the two"
- entry: ./scripts/ci/prek/check_go_sdk_serialization_conformance.py
- # golang so prek provisions the toolchain that builds the Go
serializer.
- language: golang
- pass_filenames: false
- require_serial: true
- files: >
- (?x)
- ^airflow-core/src/airflow/serialization/serialized_objects\.py$|
- ^airflow-core/src/airflow/serialization/schema\.json$|
- ^go-sdk/airflow/.*\.go$|
- ^go-sdk/schema/dag-schema\.json$|
- ^scripts/ci/lang_sdk_serialization/.*$|
- ^scripts/ci/prek/check_go_sdk_serialization_conformance\.py$
- - id: check-java-sdk-serialization-conformance
- name: Check the Java SDK serializes Dags the way Airflow does
- env:
- PYTHONPYCACHEPREFIX: .build/pycache
- description: "Serialize the shared test Dags with the Java SDK and
with Airflow, and compare the two"
- entry: ./java-sdk/scripts/ci/prek/check_serialization_conformance.py
- language: system
- pass_filenames: false
- require_serial: true
- files: >
- (?x)
- ^airflow-core/src/airflow/serialization/serialized_objects\.py$|
- ^airflow-core/src/airflow/serialization/schema\.json$|
- ^java-sdk/scripts/ci/prek/check_serialization_conformance\.py$|
- ^java-sdk/sdk/schema/dag-schema\.json$|
- ^java-sdk/sdk/build\.gradle\.kts$|
-
^java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/(Switch|Bundle|Condition|DagDef|Decider|Deps|Arg|TaskGroupRef|TriggerDagRun)\.kt$|
-
^java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/execution/Serde\.kt$|
-
^java-sdk/sdk/src/main/kotlin/org/apache/airflow/sdk/internal/(Refs|Fields)\.kt$|
- ^java-sdk/sdk/src/test/kotlin/org/apache/airflow/sdk/conformance/.*$|
- ^scripts/ci/lang_sdk_serialization/.*$
- id: check-go-version-in-sync
name: Check Go toolchain version is consistent across build files
env:
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index 4481d99b054..051fd3d5699 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -525,6 +525,15 @@ together using `pytest-xdist` (pytest-xdist distributes
the tests among parallel
* `TS SDK unit tests` (the vitest suite run by `pnpm test` in `ts-sdk/`,
exposed as the
`run-ts-sdk-tests` output) run when `TS SDK files` change: anything under
`ts-sdk/` except `.md`
files, `ts-sdk/api-docs/`, and `ts-sdk/docs/` (the docs toolchain's own
package).
+* `Lang SDK serialization conformance` (the `tests-lang-sdk-conformance` job,
which serializes the Dags
+ of `scripts/ci/lang_sdk_serialization/test_dags.yaml` with an SDK and with
Airflow and compares the
+ two) runs for each SDK listed in the `lang-sdk-conformance-sdks` output:
`go` when `Go SDK
+ conformance files` change, `java` when `Java SDK conformance files` change
and `ts` when `TS SDK
+ conformance files` change. Each group is the SDK's own files (as in `Go SDK
files`, `Java SDK files`
+ and `TS SDK files`) plus what the check compares the SDK against: Airflow's
serializer
+ (`serialized_objects.py`),
`airflow-core/src/airflow/serialization/schema.json` and the shared
+ harness under `scripts/ci/lang_sdk_serialization/`. None of those three
forces `full_tests_needed`,
+ so a change to any of them runs the check for every SDK. The list is empty
on a component branch.
* `Java SDK E2E tests` (the `java_sdk` mode of the deployed-stack tests,
exposed as the
`run-java-sdk-e2e-tests` output) run when the Java SDK sources (`java-sdk/`,
excluding `.md`), the
Java test-fixture and native-Dag bundles
(`airflow-e2e-tests/java-test-bundle/`,
@@ -614,18 +623,14 @@ when some files are not changed. Those are the rules
implemented:
`regenerate-java-sdk-verification-metadata` checks are skipped. All three
run the java-sdk
Gradle wrapper, and `regenerate-java-sdk-verification-metadata` also
resolves the whole Java
SDK dependency graph from Maven Central
- * if no `Java SDK conformance files` changed -
`check-java-sdk-serialization-conformance`
- and `sync-java-sdk-dag-schema` are skipped
(`check-java-sdk-serialization-conformance`
- compiles the Java SDK with Gradle and resolves its dependencies from Maven
Central, and
- `sync-java-sdk-dag-schema` runs the `:sdk:syncDagSchema` Gradle task). The
group is wider
- than `Java SDK files`. It also covers Airflow's serializer
(`serialized_objects.py`),
- `airflow-core/src/airflow/serialization/schema.json` and the shared
harness under
- `scripts/ci/lang_sdk_serialization/`. These files are in the group because
the conformance
- check compares the SDK against them, and none of them forces
`full_tests_needed`. Because
- airflow-core's `schema.json` is in the group, `sync-java-sdk-dag-schema`
still runs on a PR
- that changes only that file. Such a change is what leaves the Java SDK's
copy in
+ * if no `Java SDK conformance files` changed - `sync-java-sdk-dag-schema` is
skipped (it runs the
+ `:sdk:syncDagSchema` Gradle task). The group is wider than `Java SDK
files`: it also covers
+ Airflow's serializer, `airflow-core/src/airflow/serialization/schema.json`
and the shared
+ harness under `scripts/ci/lang_sdk_serialization/` (see `Lang SDK
serialization conformance`
+ above). Because airflow-core's `schema.json` is in the group,
`sync-java-sdk-dag-schema` still
+ runs on a PR that changes only that file. Such a change is what leaves the
Java SDK's copy in
`java-sdk/sdk/schema/dag-schema.json` out of date
- * whichever of the five Java SDK hooks above runs first in a job pays for
Gradle's cold
+ * whichever of the four Java SDK hooks above runs first in a job pays for
Gradle's cold
start, which includes downloading the Gradle distribution. A PR skips the
cold start only
when it changes none of the `Java SDK conformance files`
* neither Java SDK group includes `.pre-commit-config.yaml` or CI setup
files. A PR that
@@ -698,6 +703,7 @@ GitHub Actions to pass the list of parameters to a command
to execute
| kubernetes-combos-list-as-string | All combinations
of Python version and Kubernetes version to use for tests as space-separated
string | 3.11-v1.25.2 3.12-v1.28.13 | * |
| kubernetes-versions | All Kubernetes
versions to use for tests as JSON array
| \['v1.25.2'\] | |
| kubernetes-versions-list-as-string | All Kubernetes
versions to use for tests as space-separated string
| v1.25.2 | * |
+| lang-sdk-conformance-sdks | Language SDKs
whose serialization conformance check runs, as JSON array
| \["go", "java", "ts"\] | |
| latest-versions-only | If set, the number
of Python, Kubernetes, DB versions will be limited to the latest ones.
| false | |
| mypy-checks | List of folders to
be considered for mypy checks
| \["airflow_aux", "airflow_core"\] | |
| mysql-exclude | Which versions of
MySQL to exclude for tests as JSON array
| [] | |
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 614cc2672f8..868ed5bf4ff 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -147,9 +147,11 @@ class FileGroupForCi(Enum):
TASK_SDK_INTEGRATION_TEST_FILES = auto()
AGENT_FRAMEWORK_FILES = auto()
GO_SDK_FILES = auto()
+ GO_SDK_CONFORMANCE_FILES = auto()
JAVA_SDK_FILES = auto()
JAVA_SDK_CONFORMANCE_FILES = auto()
TS_SDK_FILES = auto()
+ TS_SDK_CONFORMANCE_FILES = auto()
TS_SDK_DOCS_FILES = auto()
AIRFLOW_CTL_FILES = auto()
AIRFLOW_CTL_INTEGRATION_TEST_FILES = auto()
@@ -217,6 +219,15 @@ LANG_SDK_NATIVE_DAG_PARSING_FILES = (
r"^task-sdk/src/airflow/sdk/importers/.*",
)
+# What every language SDK's serialization conformance check compares the SDK
against: Airflow's
+# serializer and schema, and the shared harness. They do not force
full_tests_needed, so each SDK's
+# conformance group lists them next to the SDK's own files.
+LANG_SDK_CONFORMANCE_SHARED_FILES = (
+ r"^airflow-core/src/airflow/serialization/serialized_objects\.py$",
+ r"^airflow-core/src/airflow/serialization/schema\.json$",
+ r"^scripts/ci/lang_sdk_serialization/.*",
+)
+
CI_FILE_GROUP_MATCHES: HashableDict[FileGroupForCi] = HashableDict(
{
FileGroupForCi.ENVIRONMENT_FILES: [
@@ -538,6 +549,12 @@ CI_FILE_GROUP_MATCHES: HashableDict[FileGroupForCi] =
HashableDict(
# everything else (go.mod, go.sum, build config) must trigger the
unit tests.
r"^go-sdk/(?!.*\.md$).*",
],
+ FileGroupForCi.GO_SDK_CONFORMANCE_FILES: [
+ # The Go SDK, plus what its serialization conformance check
compares it against:
+ # Airflow's serializer and schema, and the shared harness.
+ r"^go-sdk/(?!.*\.md$).*",
+ *LANG_SDK_CONFORMANCE_SHARED_FILES,
+ ],
FileGroupForCi.JAVA_SDK_FILES: [
# `.md` excluded — doc-only edits do not affect the Gradle build.
r"^java-sdk/(?!.*\.md$).*",
@@ -546,9 +563,7 @@ CI_FILE_GROUP_MATCHES: HashableDict[FileGroupForCi] =
HashableDict(
# The Java SDK, plus what its serialization conformance check
compares it against:
# Airflow's serializer and schema, and the shared harness.
r"^java-sdk/(?!.*\.md$).*",
- r"^airflow-core/src/airflow/serialization/serialized_objects\.py$",
- r"^airflow-core/src/airflow/serialization/schema\.json$",
- r"^scripts/ci/lang_sdk_serialization/.*",
+ *LANG_SDK_CONFORMANCE_SHARED_FILES,
],
FileGroupForCi.TS_SDK_DOCS_FILES: [
# TypeDoc renders the reference from the SDK sources and category
entry points,
@@ -568,6 +583,12 @@ CI_FILE_GROUP_MATCHES: HashableDict[FileGroupForCi] =
HashableDict(
# docs toolchain's own package, with its own tests that vitest
does not run.
r"^ts-sdk/(?!api-docs/)(?!docs/)(?!.*\.md$).*",
],
+ FileGroupForCi.TS_SDK_CONFORMANCE_FILES: [
+ # The TS SDK, plus what its serialization conformance check
compares it against:
+ # Airflow's serializer and schema, and the shared harness.
+ r"^ts-sdk/(?!api-docs/)(?!docs/)(?!.*\.md$).*",
+ *LANG_SDK_CONFORMANCE_SHARED_FILES,
+ ],
FileGroupForCi.ASSET_FILES: [
r"^airflow-core/src/airflow/assets/",
r"^airflow-core/src/airflow/models/assets/",
@@ -1510,6 +1531,19 @@ class SelectiveChecks:
def run_ts_sdk_docs(self) -> bool:
return self._should_be_run(FileGroupForCi.TS_SDK_DOCS_FILES)
+ @cached_property
+ def lang_sdk_conformance_sdks(self) -> str:
+ """JSON list of the language SDKs whose serialization conformance
check runs, as the matrix of its job."""
+ # No component branch runs it, as for the outputs scoped with
_scoped_to_component_branch.
+ if self._component_branch is not None:
+ return json.dumps([])
+ groups = {
+ "go": FileGroupForCi.GO_SDK_CONFORMANCE_FILES,
+ "java": FileGroupForCi.JAVA_SDK_CONFORMANCE_FILES,
+ "ts": FileGroupForCi.TS_SDK_CONFORMANCE_FILES,
+ }
+ return json.dumps([sdk for sdk, group in groups.items() if
self._should_be_run(group)])
+
@cached_property
@_scoped_to_component_branch
def run_airflow_ctl_tests(self) -> bool:
@@ -2144,14 +2178,10 @@ class SelectiveChecks:
# depend on that resolution.
prek_hooks_to_skip.add("regenerate-java-sdk-verification-metadata")
if not self._matching_files(FileGroupForCi.JAVA_SDK_CONFORMANCE_FILES,
CI_FILE_GROUP_MATCHES):
- # This hook compiles the Java SDK with Gradle and resolves its
dependencies from Maven
- # Central. Skip it unless a java-sdk file, Airflow's serializer or
schema, or the shared
- # harness changed. Those last do not force full_tests_needed, so
they need their own group.
- prek_hooks_to_skip.add("check-java-sdk-serialization-conformance")
- # sync-java-sdk-dag-schema runs the Gradle wrapper too. This hook
is skipped together
- # with the conformance check, not with ktlint, because
JAVA_SDK_CONFORMANCE_FILES
- # includes airflow-core's schema.json. A PR that changes only
schema.json can leave
- # the Java SDK's copy of the schema out of date, and this hook
detects the outdated copy.
+ # sync-java-sdk-dag-schema runs the Gradle wrapper too. This hook
is skipped with
+ # JAVA_SDK_CONFORMANCE_FILES, not with ktlint, because that group
includes airflow-core's
+ # schema.json. A PR that changes only schema.json can leave the
Java SDK's copy of the
+ # schema out of date, and this hook detects the outdated copy.
prek_hooks_to_skip.add("sync-java-sdk-dag-schema")
if not self._matching_files(FileGroupForCi.TS_SDK_FILES,
CI_FILE_GROUP_MATCHES):
# This hook regenerates ts-sdk/src/generated/supervisor.ts from
the vendored schema
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 2aa1c6fc5ce..5cdac3e1e94 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -117,7 +117,7 @@ LIST_OF_ALL_PROVIDER_TESTS_AS_JSON = json.dumps(
ALL_SKIPPED_COMMITS_ON_NO_CI_IMAGE = (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -131,7 +131,7 @@ ALL_SKIPPED_COMMITS_ON_NO_CI_IMAGE = (
ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED = "identity"
ALL_SKIPPED_COMMITS_IF_ONLY_UI_OPENAPI_CHANGED = (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,"
"lint-helm-chart,mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,"
"mypy-airflow-e2e-tests,mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,"
"mypy-kubernetes-tests,mypy-scripts,mypy-shared-configuration,mypy-shared-dagnode,"
@@ -143,7 +143,7 @@ ALL_SKIPPED_COMMITS_IF_ONLY_UI_OPENAPI_CHANGED = (
)
ALL_SKIPPED_COMMITS_IF_NO_UI = (
-
"check-java-sdk-serialization-conformance,check-ts-sdk-supervisor-schema,identity,ktlint,mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
+
"check-ts-sdk-supervisor-schema,identity,ktlint,mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
"mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging,"
@@ -153,7 +153,7 @@ ALL_SKIPPED_COMMITS_IF_NO_UI = (
"regenerate-java-sdk-verification-metadata,spotless,sync-java-sdk-dag-schema,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui"
)
ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS = (
-
"check-java-sdk-serialization-conformance,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+ "check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -164,7 +164,7 @@ ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS = (
)
ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS = (
-
"check-java-sdk-serialization-conformance,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+ "check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -179,7 +179,7 @@ ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS = (
# forced. airflow-core Python changed (so mypy-airflow-core + flynt run); no
# provider.yaml, helm, or UI files changed, so those checks stay skipped.
ALL_SKIPPED_COMMITS_IF_ONLY_API_SOURCE_CHANGED = (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -192,7 +192,7 @@ ALL_SKIPPED_COMMITS_IF_ONLY_API_SOURCE_CHANGED = (
)
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_AND_UI = (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,"
+ "check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -204,7 +204,7 @@ ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_AND_UI = (
)
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS = (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -217,7 +217,7 @@ ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS = (
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS = (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -229,7 +229,7 @@ ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS = (
)
ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS = (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -240,7 +240,7 @@ ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS = (
)
ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED = (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -485,7 +485,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"run-amazon-tests": "false",
"docs-build": "true",
"skip-prek-hooks": (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -531,7 +531,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"run-api-tests": "true",
"docs-build": "true",
"skip-prek-hooks": (
-
"check-java-sdk-serialization-conformance,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+
"check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -786,7 +786,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"docs-build": "true",
"full-tests-needed": "false",
"skip-prek-hooks": (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -824,7 +824,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"docs-build": "false",
"full-tests-needed": "false",
"skip-prek-hooks": (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -887,7 +887,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"docs-build": "true",
"full-tests-needed": "false",
"skip-prek-hooks": (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -923,7 +923,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"docs-build": "false",
"full-tests-needed": "false",
"skip-prek-hooks": (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -1272,7 +1272,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"docs-build": "false",
"run-kubernetes-tests": "false",
"skip-prek-hooks": (
-
"check-java-sdk-serialization-conformance,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+
"check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -1479,7 +1479,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-amazon-tests": "false",
"docs-build": "true",
"skip-prek-hooks": (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,flynt,identity,ktlint,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -1989,7 +1989,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
("shared/logging/src/airflow_shared/logging/remote.py",),
{
"skip-prek-hooks": (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-core,mypy-airflow-ctl,mypy-airflow-ctl-tests,"
"mypy-airflow-e2e-tests,mypy-dev,mypy-devel-common,mypy-docker-tests,"
"mypy-helm-tests,mypy-kubernetes-tests,mypy-scripts,"
@@ -2027,7 +2027,6 @@ def test_expected_output_pull_request_main(
"ktlint",
"spotless",
"regenerate-java-sdk-verification-metadata",
- "check-java-sdk-serialization-conformance",
"sync-java-sdk-dag-schema",
],
)
@@ -2067,7 +2066,6 @@ def
test_java_sdk_gradle_hooks_only_run_for_java_sdk_changes(
assert (hook in skipped_hooks) is hook_skipped
[email protected]("hook", ["check-java-sdk-serialization-conformance",
"sync-java-sdk-dag-schema"])
@pytest.mark.parametrize(
("files", "hook_skipped"),
[
@@ -2093,9 +2091,7 @@ def
test_java_sdk_gradle_hooks_only_run_for_java_sdk_changes(
),
],
)
-def test_java_sdk_conformance_hook_runs_for_serializer_changes(
- files: tuple[str, ...], hook_skipped: bool, hook: str
-):
+def test_sync_java_sdk_dag_schema_runs_for_serializer_changes(files:
tuple[str, ...], hook_skipped: bool):
stderr = SelectiveChecks(
files=files,
commit_ref=NEUTRAL_COMMIT,
@@ -2104,7 +2100,43 @@ def
test_java_sdk_conformance_hook_runs_for_serializer_changes(
default_branch="main",
)
skipped_hooks =
get_outputs_from_stderr(str(stderr))["skip-prek-hooks"].split(",")
- assert (hook in skipped_hooks) is hook_skipped
+ assert ("sync-java-sdk-dag-schema" in skipped_hooks) is hook_skipped
+
+
[email protected](
+ ("files", "expected_sdks"),
+ [
+ pytest.param(("go-sdk/airflow/serialize.go",), ["go"],
id="go-sdk-change"),
+ pytest.param(("java-sdk/sdk/build.gradle.kts",), ["java"],
id="java-sdk-change"),
+ pytest.param(("ts-sdk/src/coordinator/serde.ts",), ["ts"],
id="ts-sdk-change"),
+ pytest.param(
+ ("airflow-core/src/airflow/serialization/serialized_objects.py",),
+ ["go", "java", "ts"],
+ id="airflow-serializer-change",
+ ),
+ pytest.param(
+ ("airflow-core/src/airflow/serialization/schema.json",),
+ ["go", "java", "ts"],
+ id="airflow-dag-schema-change",
+ ),
+ pytest.param(
+ ("scripts/ci/lang_sdk_serialization/compare.py",),
+ ["go", "java", "ts"],
+ id="shared-harness-change",
+ ),
+ pytest.param(("go-sdk/README.md",), [], id="sdk-docs-only-change"),
+ pytest.param(("SECURITY.md",), [], id="unrelated-change"),
+ ],
+)
+def test_lang_sdk_conformance_sdks(files: tuple[str, ...], expected_sdks:
list[str]):
+ stderr = SelectiveChecks(
+ files=files,
+ commit_ref=NEUTRAL_COMMIT,
+ github_event=GithubEvents.PULL_REQUEST,
+ pr_labels=tuple(),
+ default_branch="main",
+ )
+ assert
json.loads(get_outputs_from_stderr(str(stderr))["lang-sdk-conformance-sdks"])
== expected_sdks
def test_spotless_skipped_when_only_airflow_serializer_changes():
@@ -3076,7 +3108,7 @@ def test_expected_output_fork_push(
"docs-build": "true",
"docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
"skip-prek-hooks": (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -3118,7 +3150,7 @@ def test_expected_output_fork_push(
"microsoft.mssql mongo mysql openlineage oracle postgres "
"presto salesforce samba sftp ssh standard trino",
"skip-prek-hooks": (
-
"check-java-sdk-serialization-conformance,check-ts-sdk-supervisor-schema,identity,ktlint,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
+
"check-ts-sdk-supervisor-schema,identity,ktlint,mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
"mypy-shared-configuration,mypy-shared-dagnode,mypy-shared-listeners,mypy-shared-logging,"
@@ -3163,7 +3195,7 @@ def test_expected_output_fork_push(
"docs-build": "true",
"docs-list-as-string": "apache-airflow",
"skip-prek-hooks": (
-
"check-java-sdk-serialization-conformance,check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
+
"check-provider-yaml-valid,check-ts-sdk-supervisor-schema,identity,ktlint,lint-helm-chart,"
"mypy-airflow-ctl,mypy-airflow-ctl-tests,mypy-airflow-e2e-tests,"
"mypy-dev,mypy-devel-common,mypy-docker-tests,mypy-helm-tests,mypy-kubernetes-tests,"
"mypy-scripts,"
@@ -5643,6 +5675,7 @@ def
test_component_branch_runs_only_its_component(context, expected_on, expected
assert checks.docs_list_as_string == expected_docs
assert checks.docs_build is (expected_docs is not None)
assert checks.basic_checks_only is ("ci_image_build" not in expected_on)
+ assert checks.lang_sdk_conformance_sdks == "[]"
@pytest.mark.parametrize(
diff --git a/go-sdk/DEVELOPMENT.md b/go-sdk/DEVELOPMENT.md
index b6511e8ec99..cd40e52e5b4 100644
--- a/go-sdk/DEVELOPMENT.md
+++ b/go-sdk/DEVELOPMENT.md
@@ -159,17 +159,17 @@ dag.Task(load, airflow.Inputs(transformed,
airflow.Literal("s3://bucket/out")))
A literal must be JSON that decodes into its parameter type, and it adds no
edge to the Dag.
-The `check-go-sdk-serialization-conformance` prek hook builds the Dags of
+The serialization conformance check builds the Dags of
[`scripts/ci/lang_sdk_serialization/test_dags.yaml`](../scripts/ci/lang_sdk_serialization/test_dags.yaml)
twice: with this SDK, and with Python and Airflow's own serializer. It loads
the Go output with
-Airflow's deserializer and compares the two serializations field by field. Run
it from the
-repository root:
+Airflow's deserializer and compares the two serializations field by field. CI
runs it in the
+`Lang SDK serialization conformance` job. Run it from the repository root (it
needs `go` and `uv`):
```bash
-prek run check-go-sdk-serialization-conformance --all-files
+python3 scripts/ci/lang_sdk_serialization/check_go_sdk.py
```
-The hook runs every Dag of the file, including the ones that `requires` a
feature that only some SDKs
+The check runs every Dag of the file, including the ones that `requires` a
feature that only some SDKs
have.
[`go_native.json`](../airflow-core/tests/unit/dag_processing/lang_sdk_fixtures/go_native.json)
holds the Go output for the same Dags, so that the Airflow core tests check
that Airflow loads it.
`TestSerializeConformanceDags` says how to rewrite it.
diff --git a/go-sdk/airflow/conformance_test.go
b/go-sdk/airflow/conformance_test.go
index a86b9209531..64dc043f460 100644
--- a/go-sdk/airflow/conformance_test.go
+++ b/go-sdk/airflow/conformance_test.go
@@ -37,8 +37,8 @@ import (
// TestSerializeConformanceDags builds the Dags of
scripts/ci/lang_sdk_serialization/test_dags.yaml
// with this SDK, serializes them, and writes them to a JSON file keyed by
dag_id.
// serialize_python.py in the same directory does the same with Airflow's own
serializer. compare.py
-// there runs this test as the serializer of the Go SDK for the
-// check-go-sdk-serialization-conformance prek hook, and passes the two paths
after -args:
+// there runs this test as the serializer of the Go SDK for check_go_sdk.py,
and passes the two
+// paths after -args:
//
// go -C go-sdk test ./airflow -count=1 -run
'^TestSerializeConformanceDags$' -args <test_dags.yaml> <output.json>
//
diff --git a/java-sdk/sdk/build.gradle.kts b/java-sdk/sdk/build.gradle.kts
index 825f481b950..09d626e68f6 100644
--- a/java-sdk/sdk/build.gradle.kts
+++ b/java-sdk/sdk/build.gradle.kts
@@ -959,7 +959,7 @@ publishing {
}
// Prints the classpath that runs the conformance serializer, for
-// java-sdk/scripts/ci/prek/check_serialization_conformance.py.
+// scripts/ci/lang_sdk_serialization/check_java_sdk.py.
tasks.register("printConformanceClasspath") {
dependsOn("testClasses")
// Capture early to keep compatibility to the Gradle configuration cache.
diff --git a/scripts/ci/prek/check_go_sdk_serialization_conformance.py
b/scripts/ci/lang_sdk_serialization/check_go_sdk.py
similarity index 96%
rename from scripts/ci/prek/check_go_sdk_serialization_conformance.py
rename to scripts/ci/lang_sdk_serialization/check_go_sdk.py
index 2a152cee655..6406548e18d 100755
--- a/scripts/ci/prek/check_go_sdk_serialization_conformance.py
+++ b/scripts/ci/lang_sdk_serialization/check_go_sdk.py
@@ -25,7 +25,7 @@ and compares the two serializations field by field.
Run from the repo root:
- python3 scripts/ci/prek/check_go_sdk_serialization_conformance.py
+ python3 scripts/ci/lang_sdk_serialization/check_go_sdk.py
It needs ``go`` and ``uv``. Exits 0 when the two serializations agree, 1
otherwise.
"""
diff --git a/java-sdk/scripts/ci/prek/check_serialization_conformance.py
b/scripts/ci/lang_sdk_serialization/check_java_sdk.py
similarity index 94%
rename from java-sdk/scripts/ci/prek/check_serialization_conformance.py
rename to scripts/ci/lang_sdk_serialization/check_java_sdk.py
index d6f454ec3e7..a65cecc4432 100755
--- a/java-sdk/scripts/ci/prek/check_serialization_conformance.py
+++ b/scripts/ci/lang_sdk_serialization/check_java_sdk.py
@@ -23,9 +23,7 @@ import subprocess
import sys
from pathlib import Path
-sys.path.insert(0, str(Path(__file__).resolve().parents[4] / "scripts" / "ci"
/ "prek"))
-
-from common_prek_utils import AIRFLOW_ROOT_PATH
+AIRFLOW_ROOT_PATH = Path(__file__).resolve().parents[3]
if __name__ not in ("__main__", "__mp_main__"):
raise SystemExit(
diff --git a/ts-sdk/scripts/ci/prek/check_serialization_conformance.py
b/scripts/ci/lang_sdk_serialization/check_ts_sdk.py
similarity index 92%
rename from ts-sdk/scripts/ci/prek/check_serialization_conformance.py
rename to scripts/ci/lang_sdk_serialization/check_ts_sdk.py
index 95a11280f2f..027b6def534 100755
--- a/ts-sdk/scripts/ci/prek/check_serialization_conformance.py
+++ b/scripts/ci/lang_sdk_serialization/check_ts_sdk.py
@@ -23,9 +23,7 @@ import subprocess
import sys
from pathlib import Path
-sys.path.insert(0, str(Path(__file__).resolve().parents[4] / "scripts" / "ci"
/ "prek"))
-
-from common_prek_utils import AIRFLOW_ROOT_PATH, run_command
+AIRFLOW_ROOT_PATH = Path(__file__).resolve().parents[3]
# The features of test_dags.yaml that
tests/conformance/serialize_typescript.ts can build. A Dag that
# requires another feature is left out until the builder gets it.
@@ -38,9 +36,10 @@ if __name__ not in ("__main__", "__mp_main__"):
)
if __name__ == "__main__":
- run_command(
+ subprocess.run(
["pnpm", "install", "--frozen-lockfile",
"--config.confirmModulesPurge=false"],
cwd=AIRFLOW_ROOT_PATH / "ts-sdk",
+ check=True,
)
compare = AIRFLOW_ROOT_PATH / "scripts" / "ci" / "lang_sdk_serialization"
/ "compare.py"
serializer = ["pnpm", "--dir", "ts-sdk", "exec", "tsx",
"tests/conformance/serialize_typescript.ts"]
diff --git a/scripts/ci/lang_sdk_serialization/compare.py
b/scripts/ci/lang_sdk_serialization/compare.py
index ecb27acb733..80ccc9324ae 100644
--- a/scripts/ci/lang_sdk_serialization/compare.py
+++ b/scripts/ci/lang_sdk_serialization/compare.py
@@ -35,8 +35,8 @@ features the SDK has, as a comma-separated list or ``all``,
and the copy of test
the Dags whose features are all among them. A Dag that requires nothing is
always in it, and the SDK
has to write exactly the Dags of the copy.
-Each SDK's own prek hook runs it this way, such as
-java-sdk/scripts/ci/prek/check_serialization_conformance.py for the Java SDK.
+Each SDK's check script next to this file runs it this way, such as
check_java_sdk.py for the
+Java SDK.
On a failure the files are kept, and their directory is printed.
"""
diff --git
a/scripts/tests/ci/prek/test_check_go_sdk_serialization_conformance.py
b/scripts/tests/ci/lang_sdk_serialization/test_check_go_sdk.py
similarity index 94%
rename from scripts/tests/ci/prek/test_check_go_sdk_serialization_conformance.py
rename to scripts/tests/ci/lang_sdk_serialization/test_check_go_sdk.py
index f3ce5fccc68..4ec45a64bb1 100644
--- a/scripts/tests/ci/prek/test_check_go_sdk_serialization_conformance.py
+++ b/scripts/tests/ci/lang_sdk_serialization/test_check_go_sdk.py
@@ -21,10 +21,10 @@ import subprocess
import sys
from unittest import mock
-import check_go_sdk_serialization_conformance as checker
+from ci.lang_sdk_serialization import check_go_sdk as checker
[email protected]("check_go_sdk_serialization_conformance.subprocess.run",
autospec=True)
[email protected]("ci.lang_sdk_serialization.check_go_sdk.subprocess.run",
autospec=True)
def
test_main_runs_compare_with_the_go_serializer_and_returns_its_exit_code(mock_run):
mock_run.return_value = subprocess.CompletedProcess(args=[], returncode=1)
diff --git a/ts-sdk/.pre-commit-config.yaml b/ts-sdk/.pre-commit-config.yaml
index 0a927e4dd01..9e4cb2081b7 100644
--- a/ts-sdk/.pre-commit-config.yaml
+++ b/ts-sdk/.pre-commit-config.yaml
@@ -68,22 +68,6 @@ repos:
additional_dependencies: ['[email protected]']
pass_filenames: false
require_serial: true
- - id: check-ts-sdk-serialization-conformance
- name: Check the TS SDK serializes Dags the way Airflow does
- env:
- PYTHONPYCACHEPREFIX: .build/pycache
- description: "Serialize the shared test Dags with the TS SDK and with
Airflow, and compare the two"
- entry: ./scripts/ci/prek/check_serialization_conformance.py
- language: node
- files: >
- (?x)
- ^schema/dag-schema\.json$|
- ^src/coordinator/serde\.ts$|
- ^src/sdk/dag\.ts$|
- ^tests/conformance/.*$
- additional_dependencies: ['[email protected]']
- pass_filenames: false
- require_serial: true
- id: check-ts-sdk-docs-package-version-in-sync
name: Check ts-sdk/docs package.json versions match ts-sdk/package.json
env:
diff --git a/ts-sdk/tests/conformance/serialize_typescript.ts
b/ts-sdk/tests/conformance/serialize_typescript.ts
index 9dabc16a71c..4bc13e2a88a 100644
--- a/ts-sdk/tests/conformance/serialize_typescript.ts
+++ b/ts-sdk/tests/conformance/serialize_typescript.ts
@@ -19,7 +19,7 @@
// Serializes the Dags of scripts/ci/lang_sdk_serialization/test_dags.yaml
with this SDK, as
// serialize_python.py there does with Airflow's serializer, and writes them
keyed by Dag id. The
-// check-ts-sdk-serialization-conformance hook has that directory's compare.py
run it as:
+// check_ts_sdk.py in that directory has compare.py run it as:
//
// pnpm --dir ts-sdk exec tsx tests/conformance/serialize_typescript.ts
<test_dags.yaml> <output.json>
