This is an automated email from the ASF dual-hosted git repository.
jason810496 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f881a99629c Add Go SDK capability manifest and compatibility matrix
(#71753)
f881a99629c is described below
commit f881a99629cd0676876924202b821537a9e4d493
Author: Jason(Zhe-You) Liu <[email protected]>
AuthorDate: Fri Sep 18 09:05:18 2026 +0800
Add Go SDK capability manifest and compatibility matrix (#71753)
* Add Go SDK capability manifest and compatibility matrix
* Keep Go SDK compatibility claims accurate
Airflow 3.3 does not recognize the Go SDK's current supervisor schema, and
the drift check should remain reliable across idiomatic Go constant
declarations.
* Keep Go SDK compatibility validation reliable
Go permits equivalent constant declaration forms, and source-file failures
should not turn the matrix check into an opaque traceback. Historical
capability dates also need context when a newer wire schema raises the current
runtime floor.
* Distinguish a missing/renamed schema version constant from a real mismatch
read_go_schema_version() returns None both when the SupervisorSchemaVersion
constant is gone and when its declaration no longer matches
SCHEMA_VERSION_PATTERN. Previously that fell through to the mismatch branch,
reporting "SupervisorSchemaVersion is None" and telling users to update
capabilities.yaml, which cannot fix a missing or renamed Go constant.
---
.pre-commit-config.yaml | 14 +++
go-sdk/README.md | 54 ++++++++++
go-sdk/capabilities.yaml | 119 +++++++++++++++++++++
scripts/ci/prek/lang_sdk_compat_matrix.py | 10 +-
scripts/ci/prek/update_go_sdk_readme_matrix.py | 110 +++++++++++++++++++
.../ci/prek/test_update_go_sdk_readme_matrix.py | 102 ++++++++++++++++++
6 files changed, 404 insertions(+), 5 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 23cadebddb9..ecccc745ae7 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -323,6 +323,20 @@ repos:
^\.pre-commit-config\.yaml$
pass_filenames: false
require_serial: true
+ - id: update-go-sdk-readme-matrix
+ name: Update the Go SDK compatibility matrix in go-sdk/README.md
+ entry: ./scripts/ci/prek/update_go_sdk_readme_matrix.py
+ language: python
+ files: >
+ (?x)
+ ^go-sdk/capabilities\.yaml$|
+ ^go-sdk/pkg/execution/messages\.go$|
+ ^go-sdk/README\.md$|
+ ^scripts/ci/prek/lang_sdk_compat_matrix\.py$|
+ ^scripts/ci/prek/update_go_sdk_readme_matrix\.py$
+ additional_dependencies: ['PyYAML>=6.0', 'rich>=13.6.0']
+ pass_filenames: false
+ require_serial: true
- id: check-go-example-mod-tidy
name: Check lang-SDK Go example module is tidy against the Go SDK
entry: ./scripts/ci/prek/check_go_example_mod_tidy.py
diff --git a/go-sdk/README.md b/go-sdk/README.md
index 14e44f3c531..a5d619025f6 100644
--- a/go-sdk/README.md
+++ b/go-sdk/README.md
@@ -275,6 +275,60 @@ the full range of task states, and alternate XCom backends
without implementing
- Deploy the matching Python stub Dag (above) into Airflow. There is no
separate Go worker to run: the
Airflow worker forks the bundle binary once per task instance.
+## Compatibility matrix
+
+Which Airflow TaskInstance states and capabilities this SDK supports. This
table is generated from
+[`capabilities.yaml`](capabilities.yaml); the conformance dimensions are
defined in the
+[Language SDK conformance
spec](https://github.com/apache/airflow/blob/main/contributing-docs/30_new_language_sdk.rst).
+The minimum version applies to the current SDK source, while "Since" records
when each capability
+first became available and can therefore be earlier than the current minimum.
+Do not edit the table by hand — edit `capabilities.yaml` and let the
`update-go-sdk-readme-matrix`
+prek hook regenerate it.
+
+<!-- BEGIN AUTO-GENERATED LANG-SDK COMPAT MATRIX -->
+
+*Min. Airflow version: 3.4 · supervisor schema: 2026-10-30*
+
+| Dimension | Tier | Supported | Since | Notes |
+|---|---|---|---|---|
+| **TaskInstance states** | | | | |
+| state: `success` | MUST | ✓ | 3.3 | |
+| state: `failed` | MUST | ✓ | 3.3 | |
+| state: `up_for_retry` | MUST | ✓ | 3.3 | RetryTask |
+| state: `skipped` | SHOULD | ✗ | – | runtime does not emit TaskState skipped
yet |
+| state: `deferred` | MAY | ✗ | – | runtime does not emit DeferTask yet |
+| state: `up_for_reschedule` | MAY | ✗ | – | runtime does not emit
RescheduleTask yet |
+| state: `awaiting_input` | MAY | ✗ | – | runtime does not emit AwaitInputTask
yet |
+| state: `removed` | MAY | ✓ | 3.3 | |
+| **Runtime capabilities** | | | | |
+| capability: `mixed-lang-stub-target` | MUST | ✓ | 3.3 | @task.stub |
+| capability: `task-logging` | MUST | ✓ | 3.3 | slog records streamed over the
logs socket |
+| capability: `xcom-read-write` | MUST | ✓ | 3.3 | PushXCom / GetXCom |
+| capability: `connection-read` | MUST | ✓ | 3.3 | GetConnection |
+| capability: `variable-read-write` | MUST | ✗ | – | GetVariable only; no
write over the comm socket yet |
+| capability: `self-contained-bundle` | MUST | ✓ | 3.3 | AFBNDL01 native
binary via airflow-go-pack |
+| capability: `retry-policy` | MAY | ✗ | – | no task-facing retry-policy API
yet |
+| capability: `task-state-store` | MAY | ✗ | – | no task-facing state-store
API yet |
+| capability: `asset-state-store` | MAY | ✗ | – | no task-facing state-store
API yet |
+| capability: `asset-event-emit` | MAY | ✗ | – | runtime does not emit asset
events yet |
+| capability: `asset-event-read` | MAY | ✗ | – | no task-facing asset-event
API yet |
+| **Native-Dag authoring** | | | | |
+| capability: `native-dag-authoring` | SHOULD | ✗ | – | native Dag authoring
not implemented yet |
+| capability: `task-args` | MUST † | n/a | – | |
+| capability: `dag-params` | MUST † | n/a | – | |
+| capability: `taskflow-dependencies` | MUST † | n/a | – | |
+| capability: `branching` | SHOULD † | n/a | – | |
+| capability: `dag-test` | SHOULD † | n/a | – | |
+| capability: `task-group` | MAY † | n/a | – | |
+| capability: `dynamic-task-mapping` | MAY † | n/a | – | |
+| capability: `asset-inlets-outlets` | MAY † | n/a | – | |
+| capability: `asset-scheduling` | MAY † | n/a | – | |
+| capability: `object-store` | MAY † | n/a | – | no object-storage API yet |
+
+*Marks: ✓ supported · ✗ not supported · n/a not applicable. A tier marked †
applies only when `native-dag-authoring` is supported.*
+
+<!-- END AUTO-GENERATED LANG-SDK COMPAT MATRIX -->
+
## How it works
The bundle binary speaks the coordinator protocol when Airflow launches it
with `--comm` and `--logs`.
diff --git a/go-sdk/capabilities.yaml b/go-sdk/capabilities.yaml
new file mode 100644
index 00000000000..94be96e9639
--- /dev/null
+++ b/go-sdk/capabilities.yaml
@@ -0,0 +1,119 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+sdk: go
+
+min_airflow_version: "3.4"
+
+# Keep in sync with SupervisorSchemaVersion in pkg/execution/messages.go,
which is embedded in
+# bundle metadata. The render hook fails if the two disagree.
+supervisor_schema_version: "2026-10-30"
+
+# The task runner terminates a task with SucceedTask, RetryTask, or TaskState
(failed/removed); it
+# does not yet emit skipped, DeferTask, RescheduleTask, or AwaitInputTask.
+states:
+ success:
+ supported: true
+ since: "3.3"
+ failed:
+ supported: true
+ since: "3.3"
+ up_for_retry:
+ supported: true
+ since: "3.3"
+ note: "RetryTask"
+ skipped:
+ supported: false
+ note: "runtime does not emit TaskState skipped yet"
+ deferred:
+ supported: false
+ note: "runtime does not emit DeferTask yet"
+ up_for_reschedule:
+ supported: false
+ note: "runtime does not emit RescheduleTask yet"
+ awaiting_input:
+ supported: false
+ note: "runtime does not emit AwaitInputTask yet"
+ removed:
+ supported: true
+ since: "3.3"
+
+# Runtime capabilities reflect the task-facing CoordinatorClient surface;
native-Dag authoring is
+# not implemented yet, so every native capability is unsupported.
+capabilities:
+ mixed-lang-stub-target:
+ supported: true
+ since: "3.3"
+ note: "@task.stub"
+ task-logging:
+ supported: true
+ since: "3.3"
+ note: "slog records streamed over the logs socket"
+ xcom-read-write:
+ supported: true
+ since: "3.3"
+ note: "PushXCom / GetXCom"
+ connection-read:
+ supported: true
+ since: "3.3"
+ note: "GetConnection"
+ variable-read-write:
+ supported: false
+ note: "GetVariable only; no write over the comm socket yet"
+ self-contained-bundle:
+ supported: true
+ since: "3.3"
+ note: "AFBNDL01 native binary via airflow-go-pack"
+ retry-policy:
+ supported: false
+ note: "no task-facing retry-policy API yet"
+ task-state-store:
+ supported: false
+ note: "no task-facing state-store API yet"
+ asset-state-store:
+ supported: false
+ note: "no task-facing state-store API yet"
+ asset-event-emit:
+ supported: false
+ note: "runtime does not emit asset events yet"
+ asset-event-read:
+ supported: false
+ note: "no task-facing asset-event API yet"
+ native-dag-authoring:
+ supported: false
+ note: "native Dag authoring not implemented yet"
+ task-args:
+ supported: false
+ dag-params:
+ supported: false
+ taskflow-dependencies:
+ supported: false
+ branching:
+ supported: false
+ dag-test:
+ supported: false
+ task-group:
+ supported: false
+ dynamic-task-mapping:
+ supported: false
+ asset-inlets-outlets:
+ supported: false
+ asset-scheduling:
+ supported: false
+ object-store:
+ supported: false
+ note: "no object-storage API yet"
diff --git a/scripts/ci/prek/lang_sdk_compat_matrix.py
b/scripts/ci/prek/lang_sdk_compat_matrix.py
index 7de1b3a30a9..9c8dd3ca92a 100644
--- a/scripts/ci/prek/lang_sdk_compat_matrix.py
+++ b/scripts/ci/prek/lang_sdk_compat_matrix.py
@@ -117,11 +117,11 @@ class LangSdk(TypedDict):
readme: Path
-# The registry of Language SDKs and where each one's manifest and README live.
Only the Java SDK
-# declares one so far; the Go and TypeScript entries record where theirs go
when those runtimes
-# declare their capabilities. Because of that, `capabilities_yaml` is a
declared location and not a
-# promise the file exists — a consumer walking the whole registry must check
`.exists()` before
-# calling load_capabilities().
+# The registry of Language SDKs and where each one's manifest and README live.
The Go and Java SDKs
+# declare manifests; the TypeScript entry records where its manifest goes when
that runtime declares
+# its capabilities. Because of that, `capabilities_yaml` is a declared
location and not a promise the
+# file exists — a consumer walking the whole registry must check `.exists()`
before calling
+# load_capabilities().
LANG_SDKS: list[LangSdk] = [
{
"id": "go",
diff --git a/scripts/ci/prek/update_go_sdk_readme_matrix.py
b/scripts/ci/prek/update_go_sdk_readme_matrix.py
new file mode 100755
index 00000000000..b44c5b9e3f4
--- /dev/null
+++ b/scripts/ci/prek/update_go_sdk_readme_matrix.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# /// script
+# requires-python = ">=3.10,<3.11"
+# dependencies = ["PyYAML>=6.0", "rich>=13.6.0"]
+# ///
+"""Regenerate the Go SDK compatibility table in ``go-sdk/README.md``.
+
+Renders the Markdown matrix from ``go-sdk/capabilities.yaml`` between the
AUTO-GENERATED markers.
+Exits non-zero when the file changed so the contributor re-stages it. The hook
also verifies that
+the manifest's supervisor schema version matches the Go runtime constant
embedded in bundle metadata.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from common_prek_utils import console, insert_documentation
+from lang_sdk_compat_matrix import (
+ AIRFLOW_ROOT_PATH,
+ LANG_SDKS,
+ README_MATRIX_FOOTER,
+ README_MATRIX_HEADER,
+ CapabilitiesDoc,
+ load_capabilities,
+ render_markdown_table,
+)
+
+SDK_ID = "go"
+GO_MESSAGES = AIRFLOW_ROOT_PATH / "go-sdk" / "pkg" / "execution" /
"messages.go"
+SCHEMA_VERSION_PATTERN = re.compile(
+
r'^\s*(?:const\s+)?SupervisorSchemaVersion(?:\s+[A-Za-z_]\w*(?:\.\w+)*)?\s*=\s*"(?P<version>[^"]+)"'
+)
+
+
+def read_go_schema_version() -> str | None:
+ """Read ``SupervisorSchemaVersion`` from the Go runtime source."""
+ for line in GO_MESSAGES.read_text().splitlines():
+ if match := SCHEMA_VERSION_PATTERN.match(line):
+ return match.group("version")
+ return None
+
+
+def check_schema_version(doc: CapabilitiesDoc) -> bool:
+ """Whether the manifest agrees with the schema version embedded in Go
bundle metadata."""
+ try:
+ go_version = read_go_schema_version()
+ except OSError as error:
+ console.print(f"[red]Could not read {GO_MESSAGES}: {error}[/]")
+ return False
+ if go_version is None:
+ console.print(
+ "[red]Could not read SupervisorSchemaVersion from
go-sdk/pkg/execution/messages.go: "
+ "the constant is missing, or its declaration no longer matches
this hook's pattern. "
+ "Restore the constant, or update SCHEMA_VERSION_PATTERN if the
declaration changed. "
+ "Editing go-sdk/capabilities.yaml cannot fix this.[/]"
+ )
+ return False
+ declared = doc["supervisor_schema_version"]
+ if go_version == declared:
+ return True
+ console.print(
+ f"[red]go-sdk/capabilities.yaml declares supervisor_schema_version
{declared!r} but "
+ f"pkg/execution/messages.go SupervisorSchemaVersion is {go_version!r}.
"
+ "Update capabilities.yaml to match.[/]"
+ )
+ return False
+
+
+def main() -> int:
+ sdk = next(entry for entry in LANG_SDKS if entry["id"] == SDK_ID)
+ doc = load_capabilities(sdk["capabilities_yaml"], expected_sdk=SDK_ID)
+ if not check_schema_version(doc):
+ return 1
+ changed = insert_documentation(
+ sdk["readme"],
+ render_markdown_table(doc),
+ README_MATRIX_HEADER,
+ README_MATRIX_FOOTER,
+ extra_information="the Go SDK compatibility matrix",
+ )
+ if changed:
+ console.print(
+ "[yellow]Regenerated the Go SDK compatibility matrix in
go-sdk/README.md; re-stage it.[/]"
+ )
+ return 1
+ return 0
+
+
+if __name__ in ("__main__", "__mp_main__"):
+ raise SystemExit(main())
diff --git a/scripts/tests/ci/prek/test_update_go_sdk_readme_matrix.py
b/scripts/tests/ci/prek/test_update_go_sdk_readme_matrix.py
new file mode 100644
index 00000000000..4bfdde594c8
--- /dev/null
+++ b/scripts/tests/ci/prek/test_update_go_sdk_readme_matrix.py
@@ -0,0 +1,102 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import pytest
+import yaml
+from ci.prek import lang_sdk_compat_matrix as matrix,
update_go_sdk_readme_matrix as hook
+
+SCHEMA_VERSION = "2026-06-16"
+
+
+def _doc() -> dict:
+ def entry(supported: bool) -> dict:
+ return {"supported": supported, "since": "3.3" if supported else None,
"note": ""}
+
+ return {
+ "sdk": "go",
+ "supervisor_schema_version": SCHEMA_VERSION,
+ "min_airflow_version": "3.3",
+ "states": {state: entry(True) for state, _ in matrix.STATE_DIMENSIONS},
+ "capabilities": {cap.name: entry(True) for cap in
matrix.CAPABILITY_DIMENSIONS},
+ }
+
+
+class TestMain:
+ @pytest.fixture
+ def wired(self, tmp_path, monkeypatch):
+ capabilities_yaml = tmp_path / "capabilities.yaml"
+ capabilities_yaml.write_text(yaml.safe_dump(_doc()))
+ readme = tmp_path / "README.md"
+
readme.write_text(f"intro\n\n{matrix.README_MATRIX_HEADER}\n{matrix.README_MATRIX_FOOTER}\n\noutro\n")
+ go_messages = tmp_path / "messages.go"
+ go_messages.write_text(f'package execution\n\nconst
SupervisorSchemaVersion = "{SCHEMA_VERSION}"\n')
+ monkeypatch.setattr(
+ hook,
+ "LANG_SDKS",
+ [{"id": "go", "capabilities_yaml": capabilities_yaml, "readme":
readme}],
+ )
+ monkeypatch.setattr(hook, "GO_MESSAGES", go_messages)
+ return readme
+
+ def test_generates_target_then_is_idempotent(self, wired):
+ assert hook.main() == 1
+ content = wired.read_text()
+ assert "| Dimension | Tier | Supported | Since | Notes |" in content
+ assert matrix.SUPPORTED_MARK in content
+ assert content.startswith("intro\n") and content.endswith("outro\n")
+
+ assert hook.main() == 0
+
+ def
test_schema_version_disagreeing_with_go_source_fails_without_writing(self,
wired):
+ hook.GO_MESSAGES.write_text('package execution\n\nconst
SupervisorSchemaVersion = "2020-01-01"\n')
+ assert hook.main() == 1
+ assert matrix.README_MATRIX_HEADER + "\n" +
matrix.README_MATRIX_FOOTER in wired.read_text()
+
+ def test_renamed_schema_version_constant_fails_without_writing(self,
wired):
+ hook.GO_MESSAGES.write_text(
+ 'package execution\n\nconst SupervisorSchemaVersionRenamed =
"2020-01-01"\n'
+ )
+ assert hook.main() == 1
+ assert matrix.README_MATRIX_HEADER + "\n" +
matrix.README_MATRIX_FOOTER in wired.read_text()
+
+ def test_schema_version_matches_go_source(self):
+ sdk = next(entry for entry in matrix.LANG_SDKS if entry["id"] ==
hook.SDK_ID)
+ doc = matrix.load_capabilities(sdk["capabilities_yaml"],
expected_sdk=hook.SDK_ID)
+ assert doc["supervisor_schema_version"] ==
hook.read_go_schema_version()
+
+ @pytest.mark.parametrize(
+ "declaration",
+ [
+ f'const (\n\tSupervisorSchemaVersion = "{SCHEMA_VERSION}"\n)',
+ f'const SupervisorSchemaVersion string = "{SCHEMA_VERSION}"',
+ f'const SupervisorSchemaVersion = "{SCHEMA_VERSION}" // current
schema',
+ f'const SupervisorSchemaVersion="{SCHEMA_VERSION}"',
+ ],
+ )
+ def test_reads_supported_schema_version_declarations(self, declaration,
tmp_path, monkeypatch):
+ go_messages = tmp_path / "messages.go"
+ go_messages.write_text(f"package execution\n\n{declaration}\n")
+ monkeypatch.setattr(hook, "GO_MESSAGES", go_messages)
+
+ assert hook.read_go_schema_version() == SCHEMA_VERSION
+
+ def test_missing_go_source_fails_without_writing(self, wired):
+ hook.GO_MESSAGES.unlink()
+
+ assert hook.main() == 1
+ assert matrix.README_MATRIX_HEADER + "\n" +
matrix.README_MATRIX_FOOTER in wired.read_text()
