This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 92e42ed8ab1 Stop the provider registry from publishing unreleased
providers (#76109)
92e42ed8ab1 is described below
commit 92e42ed8ab134946b9faf142d9497368e764fae3
Author: Kaxil Naik <[email protected]>
AuthorDate: Mon Nov 2 00:40:27 2026 +0000
Stop the provider registry from publishing unreleased providers (#76109)
The registry's release-tag filter never ran in CI: breeze ran
extract_metadata.py inside a container with no .git, so git found no tags
and the script fell back to every version in provider.yaml. A live build
on 2026-09-29 published common-dataquality 0.1.0, which was never released.
extract_metadata.py now runs on the host, where the tags are visible, and
skips providers with state: not-ready. A live registry build fails if the
checkout has no providers-* tags; local runs without tags still warn and
keep every version.
---
.github/workflows/registry-build.yml | 8 +-
dev/breeze/doc/11_registry_tasks.rst | 8 +-
.../airflow_breeze/commands/registry_commands.py | 26 +++++--
dev/breeze/tests/test_registry_extract_data.py | 90 ++++++++++++++++++++++
dev/registry/extract_metadata.py | 22 ++++--
dev/registry/tests/test_extract_metadata.py | 72 +++++++++++++++--
registry/README.md | 4 +
7 files changed, 203 insertions(+), 27 deletions(-)
diff --git a/.github/workflows/registry-build.yml
b/.github/workflows/registry-build.yml
index f9a9f5a4275..2e7cc9c8516 100644
--- a/.github/workflows/registry-build.yml
+++ b/.github/workflows/registry-build.yml
@@ -181,8 +181,7 @@ jobs:
ref: ${{ inputs.checkout-ref }}
# Tags drive the phantom-version filter in extract_metadata.py
# (only versions with a real `providers-<id>/<ver>` tag are
- # treated as released). Without this, the filter silently
- # falls back to `versions[0]` and ships phantom versions.
+ # treated as released). The live extract step fails if they are
missing.
fetch-tags: true
# A local action is read from the workspace, not from the ref the
workflow itself runs from,
@@ -279,6 +278,11 @@ jobs:
ALLOW_UNRELEASED=""
if [[ "${DESTINATION}" == "staging" ]]; then
ALLOW_UNRELEASED="--allow-unreleased"
+ elif [[ -z "$(git tag --list 'providers-*' | head -n 1)" ]]; then
+ # Without tags extract_metadata.py keeps every version in
provider.yaml,
+ # which would publish unreleased providers and versions to live.
+ echo "::error::No providers-* tags in the checkout; refusing to
publish the live registry."
+ exit 1
fi
if [[ -n "${PROVIDER}" ]]; then
breeze registry extract-data --python "${PYTHON_VERSION}" \
diff --git a/dev/breeze/doc/11_registry_tasks.rst
b/dev/breeze/doc/11_registry_tasks.rst
index 9be90b576b3..90ce5775ada 100644
--- a/dev/breeze/doc/11_registry_tasks.rst
+++ b/dev/breeze/doc/11_registry_tasks.rst
@@ -30,10 +30,10 @@ These are all of the available registry commands:
Extracting registry data
........................
-The ``breeze registry extract-data`` command runs the three extraction scripts
-(``extract_metadata.py``, ``extract_parameters.py``,
``extract_connections.py``)
-inside a breeze CI container where all providers are installed. This is the
same
-command used by the ``registry-build.yml`` CI workflow.
+The ``breeze registry extract-data`` command runs ``extract_metadata.py`` on
the host,
+where it can read the ``providers-*`` release tags, then
``extract_parameters.py`` and
+``extract_connections.py`` inside a breeze CI container where all providers
are installed.
+This is the same command used by the ``registry-build.yml`` CI workflow.
.. image:: ./images/output_registry_extract-data.svg
:target:
https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_registry_extract-data.svg
diff --git a/dev/breeze/src/airflow_breeze/commands/registry_commands.py
b/dev/breeze/src/airflow_breeze/commands/registry_commands.py
index 1298825a4cf..e4c9e074420 100644
--- a/dev/breeze/src/airflow_breeze/commands/registry_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/registry_commands.py
@@ -98,6 +98,27 @@ def extract_data(python: str, provider: str | None,
allow_unreleased: bool):
extra_args=(),
)
+ # extract_metadata.py runs on the host: it needs no providers installed,
and its
+ # release filter reads the providers-* git tags, which the container
cannot see
+ # because .git is not mounted there. It writes providers.json into
dev/registry/,
+ # where the container scripts below read it. --allow-unreleased only
applies here.
+ metadata_cmd = [
+ "uv",
+ "run",
+ "--project",
+ str(DEV_REGISTRY_DIR),
+ "python",
+ str(DEV_REGISTRY_DIR / "extract_metadata.py"),
+ ]
+ if provider:
+ metadata_cmd.extend(["--provider", provider])
+ if allow_unreleased:
+ metadata_cmd.append("--allow-unreleased")
+ with ci_group("Extracting registry metadata"):
+ metadata_result = run_command(metadata_cmd, check=False,
cwd=str(AIRFLOW_ROOT_PATH))
+ if metadata_result.returncode:
+ sys.exit(metadata_result.returncode)
+
build_ci_image_if_needed(command_params=shell_params)
# Install suspended providers that aren't in the CI image so runtime
@@ -106,13 +127,8 @@ def extract_data(python: str, provider: str | None,
allow_unreleased: bool):
install_cmd = f"pip install --quiet {' '.join(suspended_packages)} && " if
suspended_packages else ""
provider_flag = f" --provider '{provider}'" if provider else ""
- # --allow-unreleased only applies to extract_metadata.py (which owns the
- # version filter). The other two scripts read from providers.json and
- # don't need it.
- metadata_extra = " --allow-unreleased" if allow_unreleased else ""
command = (
f"{install_cmd}"
- f"python
dev/registry/extract_metadata.py{provider_flag}{metadata_extra} && "
f"python dev/registry/extract_parameters.py{provider_flag} && "
f"python dev/registry/extract_connections.py{provider_flag}"
)
diff --git a/dev/breeze/tests/test_registry_extract_data.py
b/dev/breeze/tests/test_registry_extract_data.py
new file mode 100644
index 00000000000..684e1eabbe3
--- /dev/null
+++ b/dev/breeze/tests/test_registry_extract_data.py
@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Unit tests for the registry extract-data command."""
+
+from __future__ import annotations
+
+import subprocess
+from unittest.mock import patch
+
+import pytest
+from click.testing import CliRunner
+
+from airflow_breeze.commands.registry_commands import DEV_REGISTRY_DIR,
extract_data
+
+MODULE = "airflow_breeze.commands.registry_commands"
+METADATA_CMD = [
+ "uv",
+ "run",
+ "--project",
+ str(DEV_REGISTRY_DIR),
+ "python",
+ str(DEV_REGISTRY_DIR / "extract_metadata.py"),
+]
+
+
[email protected]
+def mocks():
+ with (
+ patch(
+ f"{MODULE}.run_command",
+ autospec=True,
+ return_value=subprocess.CompletedProcess(args=[], returncode=0),
+ ) as run_command,
+ patch(f"{MODULE}.build_ci_image_if_needed", autospec=True),
+ patch(f"{MODULE}._get_suspended_provider_packages", autospec=True,
return_value=[]),
+ patch(
+ f"{MODULE}.execute_command_in_shell",
+ autospec=True,
+ return_value=subprocess.CompletedProcess(args=[], returncode=0),
+ ) as execute_command_in_shell,
+ patch(f"{MODULE}.bring_compose_project_down", autospec=True),
+ patch(f"{MODULE}.remove_docker_networks", autospec=True),
+ patch(f"{MODULE}.fix_ownership_using_docker", autospec=True),
+ ):
+ yield run_command, execute_command_in_shell
+
+
+def test_runs_extract_metadata_on_the_host_where_git_tags_are_visible(mocks):
+ run_command, execute_command_in_shell = mocks
+
+ result = CliRunner().invoke(extract_data, ["--provider", "amazon google"])
+
+ assert result.exit_code == 0, result.output
+ assert run_command.call_args.args[0] == [*METADATA_CMD, "--provider",
"amazon google"]
+ container_command = execute_command_in_shell.call_args.kwargs["command"]
+ assert "extract_metadata.py" not in container_command
+ assert "python dev/registry/extract_parameters.py --provider 'amazon
google' && " in container_command
+
+
+def test_allow_unreleased_is_forwarded_to_extract_metadata(mocks):
+ run_command, _ = mocks
+
+ result = CliRunner().invoke(extract_data, ["--allow-unreleased"])
+
+ assert result.exit_code == 0, result.output
+ assert run_command.call_args.args[0] == [*METADATA_CMD,
"--allow-unreleased"]
+
+
+def test_stops_before_the_container_when_extract_metadata_fails(mocks):
+ run_command, execute_command_in_shell = mocks
+ run_command.return_value = subprocess.CompletedProcess(args=[],
returncode=3)
+
+ result = CliRunner().invoke(extract_data, [])
+
+ assert result.exit_code == 3
+ execute_command_in_shell.assert_not_called()
diff --git a/dev/registry/extract_metadata.py b/dev/registry/extract_metadata.py
index 22c9efde153..b3fc14357c9 100644
--- a/dev/registry/extract_metadata.py
+++ b/dev/registry/extract_metadata.py
@@ -610,17 +610,24 @@ def main():
# Skipped entirely when --allow-unreleased is set: staging builds and
# local dev want to preview unreleased provider pages so maintainers
# can verify them before tagging.
+ skipped_unreleased: list[str] = []
if args.allow_unreleased:
release_tags: set[str] = set()
else:
release_tags = load_release_tags()
if not release_tags:
+ # registry-build.yml refuses a live build without tags; locally
this is a
+ # clone without the providers-* tags, such as a fork.
print(
" Warning: no providers-* git tags found; "
- "phantom version filter is disabled (falling back to
versions[0]). "
- "If this is a CI run, ensure the checkout step uses
fetch-tags: true."
+ "phantom version filter is disabled (falling back to
versions[0])."
)
- skipped_unreleased: list[str] = []
+ # A not-ready provider is not released yet (breeze release tooling
skips it too).
+ not_ready = {pid for pid in extraction_ids if
all_provider_yamls[pid].get("state") == "not-ready"}
+ for provider_id in sorted(not_ready):
+ print(f" Skipping {provider_id}: state is not-ready")
+ skipped_unreleased.extend(not_ready)
+ extraction_ids -= not_ready
# Pre-fetch PyPI download stats and release dates for every provider in
# parallel. This used to be ~2N sequential urlopen calls inside the loop
@@ -686,7 +693,7 @@ def main():
)
continue
else:
- # No tag information available -- fall back to old behaviour.
+ # --allow-unreleased, or no tags found: keep every version listed
in provider.yaml.
versions = list(raw_versions)
version = versions[0] if versions else "0.0.0"
@@ -851,7 +858,7 @@ def main():
if skipped_unreleased:
print(
f"\nSkipped {len(skipped_unreleased)} unreleased provider(s) "
- f"(no matching git tag): {sorted(skipped_unreleased)}"
+ f"(not-ready, or no matching git tag):
{sorted(skipped_unreleased)}"
)
# Find related providers
@@ -899,9 +906,8 @@ def main():
providers_json = validate_providers_catalog({"providers": new_providers})
# Write output files to all output directories.
- # Inside breeze, registry/ is not mounted so OUTPUT_DIR writes are lost.
- # SCRIPT_DIR (dev/registry/) is always mounted, so the other extraction
- # scripts can pick up providers.json from there.
+ # The SCRIPT_DIR (dev/registry/) copy is the first place the other
extraction
+ # scripts, which run inside breeze, look for providers.json.
output_dirs = [OUTPUT_DIR, SCRIPT_DIR]
for out_dir in output_dirs:
diff --git a/dev/registry/tests/test_extract_metadata.py
b/dev/registry/tests/test_extract_metadata.py
index bbf4aa0c716..1a666710ed1 100644
--- a/dev/registry/tests/test_extract_metadata.py
+++ b/dev/registry/tests/test_extract_metadata.py
@@ -837,19 +837,19 @@ class TestVersionsListFiltering:
# ---------------------------------------------------------------------------
@patch("extract_metadata.fetch_provider_inventory", autospec=True,
return_value=None)
@patch("extract_metadata.fetch_pypi_data_parallel", autospec=True,
return_value={})
-@patch("extract_metadata.load_release_tags", autospec=True, return_value=set())
-def _run_main_for_provider(
- _load_release_tags,
+def _run_main(
_fetch_pypi_data_parallel,
_fetch_provider_inventory,
tmp_path,
provider_yaml,
src_files,
+ release_tags=frozenset({"providers-testprov/1.0.0"}),
+ extra_argv=(),
):
- """Run main() on one provider under tmp_path and return its providers.json
entry.
+ """Run main() on one provider under tmp_path and return the written
providers.json entries.
Network and filesystem dependencies are mocked or redirected, so the
- written entry has also passed main()'s ProviderContract (extra="forbid")
+ written entries have also passed main()'s ProviderContract (extra="forbid")
validation, catching key-name drift between provider.yaml, main() and
registry_contract_models.py.
"""
@@ -870,12 +870,18 @@ def _run_main_for_provider(
patch("extract_metadata.PROVIDERS_DIR", providers_dir),
patch("extract_metadata.OUTPUT_DIR", output_dir),
patch("extract_metadata.SCRIPT_DIR", script_dir),
- patch.object(sys, "argv", ["extract_metadata.py"]),
+ patch("extract_metadata.load_release_tags", autospec=True,
return_value=set(release_tags)),
+ patch.object(sys, "argv", ["extract_metadata.py", *extra_argv]),
):
main()
- written = json.loads((output_dir / "providers.json").read_text())
- return next(p for p in written["providers"] if p["id"] == "testprov")
+ return json.loads((output_dir / "providers.json").read_text())["providers"]
+
+
+def _run_main_for_provider(tmp_path, provider_yaml, src_files):
+ """Run main() on one provider under tmp_path and return its providers.json
entry."""
+ providers = _run_main(tmp_path=tmp_path, provider_yaml=provider_yaml,
src_files=src_files)
+ return next(p for p in providers if p["id"] == "testprov")
class TestMainConnectionTypesExternalServices:
@@ -934,3 +940,53 @@ class TestMainUriSchemes:
"asset": {"handler": None, "factory": None,
"to_openlineage_converter": None},
}
]
+
+
+class TestMainUnreleasedFiltering:
+ """main() drops providers and versions that were never released, unless
told not to."""
+
+ @staticmethod
+ def _provider_yaml(state="ready", versions=("1.0.0",)):
+ return f"name: Test Provider\ndescription: A test provider.\nstate:
{state}\nversions: {list(versions)}\n"
+
+ def test_keeps_untagged_versions_when_no_release_tags_found(self,
tmp_path, capsys):
+ providers = _run_main(
+ tmp_path=tmp_path,
+ provider_yaml=self._provider_yaml(),
+ src_files={},
+ release_tags=set(),
+ )
+
+ assert [(p["id"], p["version"]) for p in providers] == [("testprov",
"1.0.0")]
+ assert "phantom version filter is disabled" in capsys.readouterr().out
+
+ def test_drops_untagged_versions(self, tmp_path):
+ providers = _run_main(
+ tmp_path=tmp_path,
+ provider_yaml=self._provider_yaml(versions=("1.1.0", "1.0.0")),
+ src_files={},
+ release_tags={"providers-testprov/1.0.0",
"providers-testprov/1.1.0rc1"},
+ )
+
+ assert [(p["version"], p["versions"]) for p in providers] ==
[("1.0.0", ["1.0.0"])]
+
+ def test_skips_not_ready_provider_even_with_release_tag(self, tmp_path):
+ providers = _run_main(
+ tmp_path=tmp_path,
+ provider_yaml=self._provider_yaml(state="not-ready"),
+ src_files={},
+ )
+
+ assert [p["id"] for p in providers] == []
+
+ @pytest.mark.parametrize("state", ["ready", "not-ready"])
+ def test_allow_unreleased_includes_untagged_provider(self, tmp_path,
state):
+ providers = _run_main(
+ tmp_path=tmp_path,
+ provider_yaml=self._provider_yaml(state=state),
+ src_files={},
+ release_tags=set(),
+ extra_argv=("--allow-unreleased",),
+ )
+
+ assert [(p["id"], p["version"]) for p in providers] == [("testprov",
"1.0.0")]
diff --git a/registry/README.md b/registry/README.md
index 154bf19dcb1..cc1c2af6261 100644
--- a/registry/README.md
+++ b/registry/README.md
@@ -103,6 +103,10 @@ gitignored. Only `exploreCategories.js`, `statsData.js`,
`latestVersionData.js`,
**`extract_metadata.py`** (runs on host) walks every
`providers/*/provider.yaml` and:
1. **Parses provider metadata** — name, description, versions, dependencies
+ - **Drops unreleased entries** — skips providers with `state: not-ready`
and keeps only
+ versions with a `providers-<id>/<version>` git tag. With no tags at all
(a fork clone),
+ it warns and keeps every version; `registry-build.yml` refuses a live
build in that case.
+ `--allow-unreleased` turns the filter off.
2. **Fetches PyPI download stats** — calls
`pypistats.org/api/packages/{name}/recent`
3. **Resolves logos** — checks `public/logos/` for matching images
4. **Determines AIP-95 lifecycle stage** — reads `lifecycle` from
`provider.yaml`
