This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-0-test by this push:
new 09ef2d46dc5 [v3-0-test] Better handling missing airflow package doc
inventories (#54464) (#54473)
09ef2d46dc5 is described below
commit 09ef2d46dc56726918f1b2220b223b6a77705cde
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Aug 15 08:59:29 2025 +0200
[v3-0-test] Better handling missing airflow package doc inventories
(#54464) (#54473)
When airflow package inventories cannot be downloaded (for example
because they were deleted) we can always rebuild them locally from
sources. This might happen for example when our "main" developer
documentation published in `apache-airflow-docs` is accidentally
or purposefully deleted (for example to refresh it from scratch).
This can also happen when we have a new provider and it has never
been published, or when we add new distribution that has not yet
had a documentation built (such as task-sdk, or airflowctl in
the future).
This PR updates the existing mechanism to:
* not only handle missing "apache-airflow-" packages but also the
other airflow packages that do not follow this naming convention
* adds better diagnostics so that if we miss to add a distribution
in the future we know what to do
(cherry picked from commit 8d311c7d874eedd8c4d82ee5865be4058a3b3cc0)
Co-authored-by: Jarek Potiuk <[email protected]>
---
.../sphinx_exts/docs_build/fetch_inventories.py | 41 ++++++++++++++++++----
1 file changed, 34 insertions(+), 7 deletions(-)
diff --git a/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
b/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
index 7975b81f6a1..ca2560662e6 100644
--- a/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
+++ b/devel-common/src/sphinx_exts/docs_build/fetch_inventories.py
@@ -99,6 +99,14 @@ def should_be_refreshed(pkg_name: str,
refresh_airflow_inventories: bool) -> boo
return False
+def is_airflow_package(pkg_name: str) -> bool:
+ """
+ Check if the package name is an Airflow package.
+ This includes the main Airflow package and any provider packages.
+ """
+ return pkg_name.startswith("apache-airflow") or pkg_name in ["helm-chart",
"docker-stack", "task-sdk"]
+
+
def fetch_inventories(clean_build: bool, refresh_airflow_inventories: bool =
False) -> list[str]:
"""Fetch all inventories for Airflow documentation packages and store in
cache."""
if clean_build:
@@ -166,20 +174,39 @@ def fetch_inventories(clean_build: bool,
refresh_airflow_inventories: bool = Fal
print(f"Result: {len(success)} success, {len(failed)} failed")
if failed:
terminate = False
+ missing_airflow_packages = False
print("Failed packages:")
for pkg_no, (pkg_name, _) in enumerate(failed, start=1):
print(f"{pkg_no}. {pkg_name}")
- if not terminate and not pkg_name.startswith("apache-airflow"):
- # For solve situation that newly created Community Provider
doesn't upload inventory yet.
- # And we terminate execution only if any error happen during
fetching
- # third party intersphinx inventories.
+ if not is_airflow_package(pkg_name):
+ print(
+ f"[yellow]Missing {pkg_name} inventory is not for an
Airflow package: "
+ f"it will terminate the execution."
+ )
terminate = True
+ else:
+ print(
+ f"[yellow]Missing {pkg_name} inventory is for an Airflow
package, "
+ f"it will be built from sources."
+ )
+ missing_airflow_packages = True
if terminate:
- print("Terminate execution.")
+ print(
+ "[red]Terminate execution[/]\n\n"
+ "[yellow]Some non-airflow inventories are missing. If missing
inventory is really "
+ "an Airflow package, please add it to the `is_airflow_package`
in `fetch_inventories.py`."
+ )
raise SystemExit(1)
-
+ if missing_airflow_packages:
+ print(f"[yellow]Failed to fetch those airflow inventories -
building them first: {failed}.")
return [pkg_name for pkg_name, status in failed]
if __name__ == "__main__":
- fetch_inventories(clean_build=len(sys.argv) > 1 and sys.argv[1] == "clean")
+ if len(sys.argv) > 1 and sys.argv[1] == "clean":
+ print("[bright_blue]Cleaning inventory cache before fetching
inventories")
+ clean_build = True
+ else:
+ print("[bright_blue]Just fetching inventories without cleaning cache")
+ clean_build = False
+ fetch_inventories(clean_build=clean_build)
