This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 43d029158b0 Add "check_files.py" check for python-client and
airflow-ctl (#58542)
43d029158b0 is described below
commit 43d029158b0818a752d764a3e6765510e661ef9f
Author: Jarek Potiuk <[email protected]>
AuthorDate: Fri Nov 21 09:39:17 2025 +0100
Add "check_files.py" check for python-client and airflow-ctl (#58542)
Co-authored-by: Amogh Desai <[email protected]>
---
dev/README_RELEASE_AIRFLOWCTL.md | 11 +++-
dev/README_RELEASE_PYTHON_CLIENT.md | 19 ++++--
dev/check_files.py | 115 +++++++++++++++++++++++++++++++-----
3 files changed, 124 insertions(+), 21 deletions(-)
diff --git a/dev/README_RELEASE_AIRFLOWCTL.md b/dev/README_RELEASE_AIRFLOWCTL.md
index 609339c45a2..40ed7c36a8a 100644
--- a/dev/README_RELEASE_AIRFLOWCTL.md
+++ b/dev/README_RELEASE_AIRFLOWCTL.md
@@ -460,8 +460,6 @@ cd asf-dist/dev/airflow
export PATH_TO_SVN=$(pwd -P)
```
-TODO: implement check in ``check_files.py``
-
### Reproducible package builds checks
For Airflow-ctl distributions we introduced a reproducible build mechanism -
which means that whoever wants
@@ -524,6 +522,15 @@ You should see output similar to:
apache_airflow_airflow_ctl-1.0.0.tar.gz:No diff found
```
+You can use `check_files.py` script to verify that all expected files are
+present in SVN. This script may help also with verifying installation of the
packages.
+
+```shell script
+cd $AIRFLOW_REPO_ROOT/dev
+uv run check_files.py airflow-ctl -v ${VERSION_RC} -p ${PATH_TO_SVN}
+```
+
+
### Licence check
This can be done with the Apache RAT tool.
diff --git a/dev/README_RELEASE_PYTHON_CLIENT.md
b/dev/README_RELEASE_PYTHON_CLIENT.md
index 2b9ea157f12..ade6e515eaa 100644
--- a/dev/README_RELEASE_PYTHON_CLIENT.md
+++ b/dev/README_RELEASE_PYTHON_CLIENT.md
@@ -421,10 +421,10 @@ cd ..
[ -d asf-dist ] || svn checkout --depth=immediates
https://dist.apache.org/repos/dist asf-dist
svn update --set-depth=infinity asf-dist/dev/airflow/clients/python
-export PATH_TO_SVN="${PWD}/asf-dist/dev/airflow/clients/python/${VERSION_RC}"
+export PATH_TO_SVN="${PWD}/asf-dist/dev/airflow/"
# Then compare the packages
-cd ${PATH_TO_SVN}
+cd ${PATH_TO_SVN}/clients/python/${VERSION_RC}
for i in ${AIRFLOW_REPO_ROOT}/dist/*
do
echo "Checking if $(basename $i) is the same as $i"
@@ -439,6 +439,15 @@ In case the files are different, you should see:
Binary files apache_airflow-client-2.9.0.tar.gz and
.../apache_airflow-2.9.0.tar.gz differ
```
+You can use `check_files.py` script to verify that all expected files are
+present in SVN. This script may help also with verifying installation of the
packages.
+
+```shell script
+cd $AIRFLOW_REPO_ROOT/dev
+uv run check_files.py python-client -v ${VERSION_RC} -p ${PATH_TO_SVN}
+```
+
+
### Licence check
This can be done with the Apache RAT tool.
@@ -454,7 +463,7 @@ wget -qO-
https://dlcdn.apache.org//creadur/apache-rat-0.17/apache-rat-0.17-bin.
Unpack the release source archive (the `<package + version>-source.tar.gz`
file) to a folder
```shell script
-rm -rf /tmp/apache/airflow-python-client-src && mkdir -p
/tmp/apache-airflow-python-client-src && tar -xzf
${PATH_TO_SVN}/apache_airflow_python_client-*-source.tar.gz --strip-components
1 -C /tmp/apache-airflow-python-client-src
+rm -rf /tmp/apache/airflow-python-client-src && mkdir -p
/tmp/apache-airflow-python-client-src && tar -xzf
${PATH_TO_SVN}/clients/python/${VERSION_RC}/apache_airflow_python_client-*-source.tar.gz
--strip-components 1 -C /tmp/apache-airflow-python-client-src
```
Run the check:
@@ -528,7 +537,7 @@ gpg --keyserver keys.gnupg.net --receive-keys
CDE15C6E4D3A8EC4ECF4BA4B6674E08AD7
Once you have the keys, the signatures can be verified by running this:
```shell script
-cd ${PATH_TO_SVN}
+cd ${PATH_TO_SVN}/clients/python/${VERSION_RC}
for i in *.asc
do
echo -e "Checking $i\n"; gpg --verify $i
@@ -568,7 +577,7 @@ Primary key fingerprint: 1271 7556 040E EF2E EAF1 B9C2
75FC CD0A 25FA 0E4B
Run this:
```shell script
-cd ${PATH_TO_SVN}
+cd ${PATH_TO_SVN}/clients/python/${VERSION_RC}
for i in *.sha512
do
echo "Checking $i"; shasum -a 512 `basename $i .sha512 ` | diff - $i
diff --git a/dev/check_files.py b/dev/check_files.py
index 984db26c100..ab1bfe9b40f 100644
--- a/dev/check_files.py
+++ b/dev/check_files.py
@@ -54,6 +54,22 @@ RUN pip install "apache-airflow-task-sdk=={}"
"""
+AIRFLOW_CTL_DOCKER = """\
+FROM python:3.10
+
+# Install airflow-ctl
+RUN pip install "apache-airflow-ctl=={}"
+
+"""
+
+PYTHON_CLIENT_DOCKER = """\
+FROM python:3.10
+
+# Install python-client
+RUN pip install "apache-airflow-python-client=={}"
+
+"""
+
DOCKER_UPGRADE = """\
FROM apache/airflow:1.10.15
@@ -179,22 +195,50 @@ def expand_name_variations(files):
return sorted(base + suffix for base, suffix in itertools.product(files,
["", ".asc", ".sha512"]))
+def check_airflow_ctl_release(files: list[str], version: str):
+ print(f"Checking airflow-ctl release for version {version}:\n")
+ version = strip_rc_suffix(version)
+
+ expected_files = expand_name_variations(
+ [
+ f"apache_airflow_ctl-{version}-source.tar.gz",
+ f"apache_airflow_ctl-{version}.tar.gz",
+ f"apache_airflow_ctl-{version}-py3-none-any.whl",
+ ]
+ )
+ return check_all_files(expected_files=expected_files, actual_files=files)
+
+
+def check_python_client_release(files: list[str], version: str):
+ print(f"Checking python-client release for version {version}:\n")
+ version = strip_rc_suffix(version)
+
+ expected_files = expand_name_variations(
+ [
+ f"apache_airflow_python_client-{version}-source.tar.gz",
+ f"apache_airflow_client-{version}.tar.gz",
+ f"apache_airflow_client-{version}-py3-none-any.whl",
+ ]
+ )
+ return check_all_files(expected_files=expected_files, actual_files=files)
+
+
def check_upgrade_check(files: list[str], version: str):
print(f"Checking upgrade_check for version {version}:\n")
version = strip_rc_suffix(version)
expected_files = expand_name_variations(
[
- f"apache-airflow-upgrade-check-{version}-bin.tar.gz",
f"apache-airflow-upgrade-check-{version}-source.tar.gz",
+ f"apache-airflow-upgrade-check-{version}-bin.tar.gz",
f"apache_airflow_upgrade_check-{version}-py2.py3-none-any.whl",
]
)
return check_all_files(expected_files=expected_files, actual_files=files)
-def warn_of_missing_files(files):
- print("[red]Check failed. Here are the files we expected but did not
find:[/red]\n")
+def warn_of_missing_files(files: list[str], directory: str):
+ print(f"[red]Check failed. Here are the files we expected but did not find
in {directory}:[/red]\n")
for file in files:
print(f" - [red]{file}[/red]")
@@ -228,8 +272,11 @@ def cli():
Example usages:
python check_files.py airflow -p ~/code/airflow_svn -v 1.10.15rc1
+ python check_files.py task-sdk -p ~/code/airflow_svn -v 1.0.0rc1
+ python check_files.py airflow-ctl -p ~/code/airflow_svn -v 0.1.0rc1
+ python check_files.py python-client -p ~/code/airflow_svn -v 2.10.0rc1
python check_files.py upgrade_check -p ~/code/airflow_svn -v 1.3.0rc2
- python check_files.py providers -p ~/code/airflow_svn
+ python check_files.py providers -p ~/code/airflow_svn --release-date
2024-01-01
"""
@@ -244,14 +291,15 @@ def cli():
)
@click.pass_context
def providers(ctx, path: str, release_date: str):
- files = os.listdir(os.path.join(path, "providers", release_date))
+ directory = os.path.join(path, "providers", release_date)
+ files = os.listdir(directory)
pips = [f"{name}=={version}" for name, version in get_packages()]
missing_files = check_providers(files, release_date)
create_docker(
PROVIDERS_DOCKER.format("RUN uv pip install --pre --system " + "
".join(f"'{p}'" for p in pips))
)
if missing_files:
- warn_of_missing_files(missing_files)
+ warn_of_missing_files(missing_files, directory)
@click.command()
@@ -259,11 +307,12 @@ def providers(ctx, path: str, release_date: str):
@version_option
@click.pass_context
def airflow(ctx, path: str, version: str):
- files = os.listdir(os.path.join(path, version))
+ directory = os.path.join(path, version)
+ files = os.listdir(directory)
missing_files = check_airflow_release(files, version)
create_docker(AIRFLOW_DOCKER.format(version))
if missing_files:
- warn_of_missing_files(missing_files)
+ warn_of_missing_files(missing_files, directory)
return
@@ -272,11 +321,12 @@ def airflow(ctx, path: str, version: str):
@version_option
@click.pass_context
def task_sdk(ctx, path: str, version: str):
- files = os.listdir(os.path.join(path, version))
+ directory = os.path.join(path, version)
+ files = os.listdir(directory)
missing_files = check_task_sdk_release(files, version)
create_docker(TASK_SDK_DOCKER.format(version))
if missing_files:
- warn_of_missing_files(missing_files)
+ warn_of_missing_files(missing_files, directory)
return
@@ -285,18 +335,49 @@ def task_sdk(ctx, path: str, version: str):
@version_option
@click.pass_context
def upgrade_check(ctx, path: str, version: str):
- files = os.listdir(os.path.join(path, "upgrade-check", version))
+ directory = os.path.join(path, "upgrade-check", version)
+ files = os.listdir(directory)
missing_files = check_upgrade_check(files, version)
create_docker(DOCKER_UPGRADE.format(version))
if missing_files:
- warn_of_missing_files(missing_files)
+ warn_of_missing_files(missing_files, directory)
+ return
+
+
[email protected](name="airflow-ctl")
+@path_option
+@version_option
[email protected]_context
+def airflow_ctl(ctx, path: str, version: str):
+ directory = os.path.join(path, "airflow-ctl", version)
+ files = os.listdir(directory)
+ missing_files = check_airflow_ctl_release(files, version)
+ create_docker(AIRFLOW_CTL_DOCKER.format(version))
+ if missing_files:
+ warn_of_missing_files(missing_files, directory)
+ return
+
+
[email protected](name="python-client")
+@path_option
+@version_option
[email protected]_context
+def python_client(ctx, path: str, version: str):
+ directory = os.path.join(path, "clients", "python", version)
+ files = os.listdir(directory)
+ missing_files = check_python_client_release(files, version)
+ create_docker(PYTHON_CLIENT_DOCKER.format(version))
+ if missing_files:
+ warn_of_missing_files(missing_files, directory)
return
cli.add_command(providers)
cli.add_command(airflow)
cli.add_command(task_sdk)
+cli.add_command(airflow_ctl)
+cli.add_command(python_client)
cli.add_command(upgrade_check)
if __name__ == "__main__":
@@ -356,6 +437,9 @@ def test_check_providers_pass(monkeypatch, tmp_path):
)
files = [
+ "apache_airflow_providers-2024-01-01-source.tar.gz",
+ "apache_airflow_providers-2024-01-01-source.tar.gz.asc",
+ "apache_airflow_providers-2024-01-01-source.tar.gz.sha512",
"apache_airflow_providers_airbyte-3.1.0.tar.gz",
"apache_airflow_providers_airbyte-3.1.0.tar.gz.asc",
"apache_airflow_providers_airbyte-3.1.0.tar.gz.sha512",
@@ -369,7 +453,7 @@ def test_check_providers_pass(monkeypatch, tmp_path):
"apache_airflow_providers_foo_bar-9.6.42-py3-none-any.whl.asc",
"apache_airflow_providers_foo_bar-9.6.42-py3-none-any.whl.sha512",
]
- assert check_providers(files) == []
+ assert check_providers(files, release_date="2024-01-01") == []
def test_check_providers_failure(monkeypatch, tmp_path):
@@ -380,12 +464,15 @@ def test_check_providers_failure(monkeypatch, tmp_path):
)
files = [
+ "apache_airflow_providers-2024-02-01-source.tar.gz",
+ "apache_airflow_providers-2024-02-01-source.tar.gz.asc",
+ "apache_airflow_providers-2024-02-01-source.tar.gz.sha512",
"apache_airflow_providers_spam_egg-1.2.3.tar.gz",
"apache_airflow_providers_spam_egg-1.2.3.tar.gz.sha512",
"apache_airflow_providers_spam_egg-1.2.3-py3-none-any.whl",
"apache_airflow_providers_spam_egg-1.2.3-py3-none-any.whl.asc",
]
- assert sorted(check_providers(files)) == [
+ assert sorted(check_providers(files, release_date="2024-02-01")) == [
"apache_airflow_providers_spam_egg-1.2.3-py3-none-any.whl.sha512",
"apache_airflow_providers_spam_egg-1.2.3.tar.gz.asc",
]
