jason810496 commented on code in PR #71757:
URL: https://github.com/apache/airflow/pull/71757#discussion_r4346455199
##########
scripts/ci/prek/lang_sdk_compat_matrix.py:
##########
@@ -142,11 +208,13 @@ class LangSdk(TypedDict):
]
VALID_SDK_IDS = {sdk["id"] for sdk in LANG_SDKS}
+SDK_DISPLAY_NAMES = {"go": "Go", "java": "Java", "ts": "TypeScript"}
Review Comment:
Done in df39754: `LangSdk` carries `display_name` and `SDK_DISPLAY_NAMES` is
gone.
##########
scripts/ci/prek/update_lang_sdk_central_matrix.py:
##########
@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# /// script
+# requires-python = ">=3.11,<3.12"
+# dependencies = ["PyYAML>=6.0", "rich>=13.6.0"]
+# ///
+"""Regenerate the consolidated Language SDK compatibility matrix in the
user-facing docs.
+
+Reads every ``<sdk>/capabilities.yaml`` listed in the ``LANG_SDKS`` registry
(an SDK that has
+not published one yet simply shows the "absent" mark) and rewrites the
``list-table`` blocks in
+``airflow-core/docs/authoring-and-scheduling/language-sdks/index.rst`` between
the
+AUTO-GENERATED markers. Exits non-zero when the file changed so the
contributor re-stages it.
+"""
+
+from __future__ import annotations
+
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Final, NamedTuple
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import lang_sdk_compat_matrix as matrix
+from common_prek_utils import console, insert_documentation
+
+CENTRAL_MATRIX_HEADER: Final = ".. BEGIN AUTO-GENERATED LANG-SDK COMPAT MATRIX"
+CENTRAL_MATRIX_FOOTER: Final = ".. END AUTO-GENERATED LANG-SDK COMPAT MATRIX"
+ABSENT_MARK: Final = matrix.NO_VERSION_MARK
+
+INDEX_RST: Final = (
+ matrix.AIRFLOW_ROOT_PATH
+ / "airflow-core"
+ / "docs"
+ / "authoring-and-scheduling"
+ / "language-sdks"
+ / "index.rst"
+)
+
+
+class SdkCapabilities(NamedTuple):
+ display_name: str
+ doc: matrix.CapabilitiesDoc | None
+
+
+def load_sdk_capabilities(sdks: Sequence[matrix.LangSdk]) ->
list[SdkCapabilities]:
+ """Load every published manifest in registry order."""
+ return [
+ SdkCapabilities(
+ display_name=matrix.SDK_DISPLAY_NAMES[sdk["id"]],
+ doc=(
+ matrix.load_capabilities(sdk["capabilities_yaml"],
expected_sdk=sdk["id"])
Review Comment:
Done in df39754. `load_capabilities` now returns a `Capabilities` whose
`entries` are keyed `(name, mode)`, with the mode set only for a shared
capability (`MatrixRow.entry_key`). `CapabilitiesDoc`, the union and all three
casts are gone, and lookups are `doc.get_entry(row)`. `validate_capabilities`
became `parse_capabilities`.
##########
scripts/ci/prek/lang_sdk_compat_matrix.py:
##########
@@ -88,30 +93,91 @@ class Capability(NamedTuple):
Capability("asset-state-store", "MAY", "runtime", False),
Capability("asset-event-emit", "MAY", "runtime", False),
Capability("asset-event-read", "MAY", "runtime", False),
+ Capability("object-store", "MAY", "runtime", False),
Capability(NATIVE_DAG_GATE, "SHOULD", "native", False),
+ Capability("dag-args", "MUST", "native", True),
Capability("task-args", "MUST", "native", True),
Capability("dag-params", "MUST", "native", True),
+ Capability("explicit-dependencies", "MUST", "native", True),
Capability("taskflow-dependencies", "MUST", "native", True),
- Capability("branching", "SHOULD", "native", True),
- Capability("dag-test", "SHOULD", "native", True),
Capability("task-group", "MAY", "native", True),
- Capability("dynamic-task-mapping", "MAY", "native", True),
- Capability("asset-inlets-outlets", "MAY", "native", True),
+ Capability("if-else-branching", "SHOULD", "native", True),
+ Capability("switch-case-branching", "SHOULD", "native", True),
+ Capability("trigger-dag-run", "MAY", "native", True),
+ Capability("dynamic-dag-generation", "MAY", "native", True),
+ Capability("human-in-the-loop", "MAY", "native", True),
+ Capability("dag-test", "SHOULD", "native", True),
Capability("asset-scheduling", "MAY", "native", True),
- Capability("object-store", "MAY", "native", True),
+ Capability("mixed-lang-stub-target", "MUST", "stub", False),
+ Capability("taskflow-binding", "MUST", "stub", False),
+ Capability("dynamic-task-mapping", "MAY", "shared", True),
+ Capability("asset-inlets-outlets", "MAY", "shared", True),
]
CAPABILITY_NAMES = {cap.name for cap in CAPABILITY_DIMENSIONS}
-
-GROUP_LABELS = {"runtime": "Runtime capabilities", "native": "Native-Dag
authoring"}
-STATES_GROUP_LABEL = "TaskInstance states"
+SHARED_CAPABILITY_NAMES = {cap.name for cap in CAPABILITY_DIMENSIONS if
cap.group == "shared"}
LEGEND = (
f"Marks: {SUPPORTED_MARK} supported · {UNSUPPORTED_MARK} not supported · "
f"{NA_MARK} not applicable. A tier marked † applies only when
`{NATIVE_DAG_GATE}` is supported."
)
+class MatrixRow(NamedTuple):
+ kind: str # "state" or "capability"
Review Comment:
Done in df39754: `RowKind`, `Mode`, `Tier` and `Group` are `Literal`s, with
`Final` constants so the literal types survive.
##########
contributing-docs/30_new_language_sdk.rst:
##########
@@ -524,11 +559,27 @@ hand-authored ``<sdk>/capabilities.yaml``, from which a
prek hook generates its
|
+--> java-sdk/README.md (contributor-facing)
+--> java-sdk/sdk/module.md (Dokka -> the published API reference)
+ +--> airflow-core/docs/authoring-and-scheduling/language-sdks/index.rst
+ (consolidated cross-SDK matrix)
-The hook rewrites its targets and exits non-zero when either was stale, so a
drifted table fails
+Each hook rewrites its targets and exits non-zero when a target was stale, so
a drifted table fails
the build. Keep the manifest out of whatever the SDK publishes — it describes
the SDK rather than
being part of it; for Java that means sitting above every subproject in
``settings.gradle.kts``.
+A supported dimension may also record ``sdk_version``, the SDK's own release
that shipped it,
+alongside ``since`` (the Airflow version). It's optional; omit it when no SDK
release has been
+tagged yet. A shared capability holds one entry per authoring mode:
Review Comment:
Applied in df39754.
##########
airflow-core/docs/authoring-and-scheduling/language-sdks/index.rst:
##########
@@ -204,6 +204,229 @@ Both settings can be supplied as environment variables
using the standard Airflo
AIRFLOW__SDK__COORDINATORS='{"my-coordinator": {...}}'
AIRFLOW__SDK__QUEUE_TO_COORDINATOR='{"jdk17": "my-coordinator"}'
+Compatibility matrix
+--------------------
+
+Language SDKs mature independently, so they do not all support the same
features yet. The tables below show
+what each Language SDK supports: the features common to both authoring modes,
the features of a
+:ref:`Dag definition <language-sdks/dag-definition>`, and the features of a
+:ref:`Python Dag with stub TaskHandler <language-sdks/stub-taskhandler>`. A
supported feature shows the SDK
+release that shipped it. See the legend below the tables for what each mark
means.
+
+.. BEGIN AUTO-GENERATED LANG-SDK COMPAT MATRIX
+
+.. list-table:: Language SDK versions
+ :header-rows: 1
+ :widths: 30 15 15 15
+
+ * - Version
+ - Go
+ - Java
+ - TypeScript
+ * - Min. Airflow version
+ - 3.4.0
+ - 3.4.0
+ - 3.4.0
+ * - Current Supervisor Schema Version
+ - 2026-10-30
+ - 2026-10-30
+ - 2026-10-30
+
+.. list-table:: Common to both authoring modes
+ :header-rows: 1
+ :widths: 30 15 15 15
+
+ * - Dimension
+ - Go
+ - Java
+ - TypeScript
+ * - **TaskInstance states**
+ -
+ -
+ -
+ * - ``success`` (MUST)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``failed`` (MUST)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``up_for_retry`` (MUST)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``skipped`` (SHOULD)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``deferred`` (MAY)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``up_for_reschedule`` (MAY)
+ - ✗
+ - ✗
+ - ✗
+ * - ``awaiting_input`` (MAY)
+ - ✗
+ - ✗
+ - ✓ 1.0.0
+ * - ``removed`` (MAY)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - **Task runtime**
+ -
+ -
+ -
+ * - ``task-logging`` (MUST)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``xcom-read-write`` (MUST)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``connection-read`` (MUST)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``variable-read-write`` (MUST)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``self-contained-bundle`` (MUST)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``retry-policy`` (MAY)
+ - ✗
+ - ✗
+ - ✗
+ * - ``task-state-store`` (MAY)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``asset-state-store`` (MAY)
+ - ✗
+ - ✗
+ - ✗
+ * - ``asset-event-emit`` (MAY)
+ - ✗
+ - ✗
+ - ✗
+ * - ``asset-event-read`` (MAY)
+ - ✗
+ - ✗
+ - ✗
+ * - ``object-store`` (MAY)
+ - ✗
+ - ✗
+ - ✗
+
+.. list-table:: Dag definition
+ :header-rows: 1
+ :widths: 30 15 15 15
+
+ * - Dimension
+ - Go
+ - Java
+ - TypeScript
+ * - ``native-dag-authoring`` (SHOULD)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``dag-args`` (MUST †)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``task-args`` (MUST †)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``dag-params`` (MUST †)
+ - ✗
+ - ✗
+ - ✗
+ * - ``explicit-dependencies`` (MUST †)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``taskflow-dependencies`` (MUST †)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``task-group`` (MAY †)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``if-else-branching`` (SHOULD †)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``switch-case-branching`` (SHOULD †)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``trigger-dag-run`` (MAY †)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``dynamic-dag-generation`` (MAY †)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``human-in-the-loop`` (MAY †)
+ - ✗
+ - ✗
+ - ✓ 1.0.0
+ * - ``dag-test`` (SHOULD †)
+ - ✗
+ - ✗
+ - ✗
+ * - ``asset-scheduling`` (MAY †)
+ - ✗
+ - ✗
+ - ✗
+ * - ``dynamic-task-mapping`` (MAY †)
+ - ✗
+ - ✗
+ - ✗
+ * - ``asset-inlets-outlets`` (MAY †)
+ - ✗
+ - ✗
+ - ✗
+
+.. list-table:: Python Dag with stub TaskHandler
+ :header-rows: 1
+ :widths: 30 15 15 15
+
+ * - Dimension
+ - Go
+ - Java
+ - TypeScript
+ * - ``mixed-lang-stub-target`` (MUST)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``taskflow-binding`` (MUST)
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ - ✓ 1.0.0
+ * - ``dynamic-task-mapping`` (MAY)
+ - ✗
+ - ✗
+ - ✗
+ * - ``asset-inlets-outlets`` (MAY)
+ - ✗
+ - ✗
+ - ✗
+
+*Marks:* ``✓`` supported (with the SDK release that shipped it, when known) ·
``✗`` not supported · ``n/a`` not applicable · ``–`` not published. A tier
marked ``†`` applies only when ``native-dag-authoring`` is supported.
Review Comment:
Done in df39754: not published is `n/p`. `–` stays the no-version
placeholder in the per-SDK tables, where it does mean a missing version rather
than not applicable.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
