This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7dab711509a Fix building and testing sdist packages for Airflow
(#48111)
7dab711509a is described below
commit 7dab711509ac3229592bf6022beebf8173e7b886
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Mar 23 16:45:29 2025 +0100
Fix building and testing sdist packages for Airflow (#48111)
There were two teething problems with building and testing the
sdist packages of airflow.
1) The sdist package for apache-airflow could not be built as it
missed few files.
2) This was because testing of sdist packages skipped testing of
the apache-airflow packages by mistake
3) The sdist packages for airflow-core were not building correctly with
local hatch builds (they were produced in a airflow-core/dist and
not moved to dist folder in root.
This PR fixes all those problems.
---
.../commands/release_management_commands.py | 30 ++++++-----
pyproject.toml | 7 +++
.../run_prepare_airflow_distributions.py | 58 ++++++++++++----------
3 files changed, 55 insertions(+), 40 deletions(-)
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 6276bfcb8fb..0347b095e5c 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -305,22 +305,24 @@ class DistributionPackageInfo(NamedTuple):
build_type: Literal["airflow", "providers", "task-sdk"],
) -> tuple[DistributionPackageInfo, ...]:
if build_type == "airflow":
- default_glob_pattern = "apache[_-]airflow-[0-9]"
+ default_glob_patterns = ["apache_airflow-", "apache_airflow_core-"]
elif build_type == "task-sdk":
- default_glob_pattern = "apache[_-]airflow[_-]task[_-]sdk"
+ default_glob_patterns = ["apache_airflow_task_sdk"]
else:
- default_glob_pattern = "apache[_-]airflow[_-]providers"
+ default_glob_patterns = ["apache_airflow_providers"]
dists_info = []
if distribution_format in ["sdist", "both"]:
- for file in dist_directory.glob(f"{default_glob_pattern}*tar.gz"):
- if not file.is_file() or "-source.tar.gz" in file.name:
- continue
- dists_info.append(cls.from_sdist(filepath=file))
+ for default_glob_pattern in default_glob_patterns:
+ for file in
dist_directory.glob(f"{default_glob_pattern}*.tar.gz"):
+ if not file.is_file() or "-source.tar.gz" in file.name:
+ continue
+ dists_info.append(cls.from_sdist(filepath=file))
if distribution_format in ["wheel", "both"]:
- for file in dist_directory.glob(f"{default_glob_pattern}*whl"):
- if not file.is_file():
- continue
- dists_info.append(cls.from_wheel(filepath=file))
+ for default_glob_pattern in default_glob_patterns:
+ for file in
dist_directory.glob(f"{default_glob_pattern}*.whl"):
+ if not file.is_file():
+ continue
+ dists_info.append(cls.from_wheel(filepath=file))
return tuple(sorted(dists_info, key=lambda di: (di.package,
di.dist_type)))
def __str__(self):
@@ -473,8 +475,6 @@ def
apply_distribution_format_to_hatch_command(build_command: list[str], distrib
def _build_airflow_packages_with_hatch(
distribution_format: str, source_date_epoch: int, version_suffix_for_pypi:
str
):
- build_airflow_command = ["hatch", "build", "-c"]
- apply_distribution_format_to_hatch_command(build_airflow_command,
distribution_format)
env_copy = os.environ.copy()
env_copy["SOURCE_DATE_EPOCH"] = str(source_date_epoch)
build_airflow_core_command = ["hatch", "build", "-c", "-t", "custom"]
@@ -492,6 +492,8 @@ def _build_airflow_packages_with_hatch(
cwd=AIRFLOW_CORE_ROOT_PATH,
)
get_console().print(f"[bright_blue]Building apache-airflow distributions:
{distribution_format}\n")
+ build_airflow_command = ["hatch", "build", "-c"]
+ apply_distribution_format_to_hatch_command(build_airflow_command,
distribution_format)
with package_version(
version_suffix=version_suffix_for_pypi,
package_path=AIRFLOW_CORE_ROOT_PATH,
@@ -503,6 +505,8 @@ def _build_airflow_packages_with_hatch(
env=env_copy,
cwd=AIRFLOW_ROOT_PATH,
)
+ for distribution_path in (AIRFLOW_CORE_ROOT_PATH /
"dist").glob("apache_airflow_core*"):
+ shutil.move(distribution_path, AIRFLOW_DIST_PATH)
def _check_sdist_to_wheel_dists(dists_info: tuple[DistributionPackageInfo,
...]):
diff --git a/pyproject.toml b/pyproject.toml
index 2dfeb5efc76..e3e8c6ab996 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -140,6 +140,13 @@ path = "./hatch_build.py"
[tool.hatch.build.targets.sdist]
exclude = ["*"]
+# This force-include will be removed once we switch to `uv-only` development
environment and remove
+# hatch_build.py altogether
+[tool.hatch.build.targets.sdist.force-include]
+"airflow-core/pyproject.toml" = "airflow-core/pyproject.toml"
+"generated/provider_dependencies.json" = "generated/provider_dependencies.json"
+"airflow-core/src/airflow/__init__.py" = "airflow-core/src/airflow/__init__.py"
+
[tool.hatch.build.targets.wheel]
bypass-selection = true
diff --git a/scripts/in_container/run_prepare_airflow_distributions.py
b/scripts/in_container/run_prepare_airflow_distributions.py
index 2daaf06fd0c..e5e40424309 100755
--- a/scripts/in_container/run_prepare_airflow_distributions.py
+++ b/scripts/in_container/run_prepare_airflow_distributions.py
@@ -31,6 +31,7 @@ from rich.console import Console
console = Console(color_system="standard", width=200)
AIRFLOW_ROOT_PATH = Path(__file__).parents[2].resolve()
+AIRFLOW_DIST_PATH = AIRFLOW_ROOT_PATH / "dist"
REPRODUCIBLE_BUILD_YAML_PATH = AIRFLOW_ROOT_PATH / "reproducible_build.yaml"
AIRFLOW_CORE_ROOT_PATH = AIRFLOW_ROOT_PATH / "airflow-core"
AIRFLOW_CORE_SOURCES_PATH = AIRFLOW_CORE_ROOT_PATH / "src"
@@ -113,32 +114,35 @@ def build_airflow_packages(distribution_format: str):
sys.exit(build_process.returncode)
if distribution_format in ["both", "sdist"]:
console.print("[bright_blue]Checking if sdist packages can be built
into wheels")
- for file in (AIRFLOW_ROOT_PATH /
"dist").glob("apache_airflow_(core)?[0-9]*.tar.gz"):
- console.print(f"[bright_blue]Validate build wheel from sdist:
{file.name}")
- if "-sources.tar.gz" not in file.name:
- # no need to delete - we are in temporary container
- tmpdir = mkdtemp()
- result = subprocess.run(
- [
- sys.executable,
- "-m",
- "pip",
- "wheel",
- "--wheel-dir",
- tmpdir,
- "--no-deps",
- "--no-cache",
- "--no-binary",
- ":all:",
- file.as_posix(),
- ],
- check=False,
- )
- if result.returncode != 0:
- console.print(f"[red]Error installing {file.name}")
- sys.exit(result.returncode)
- console.print(f"[green]Sdist package {file.name} can be built
into wheels")
- console.print("[green]Sdist package is installed successfully.")
+ for glob_pattern in ["apache_airflow_core-*.tar.gz",
"apache_airflow-*.tar.gz"]:
+ for sdist_distribution_file in
AIRFLOW_DIST_PATH.glob(glob_pattern):
+ console.print(f"[bright_blue]Validate build wheel from sdist:
{sdist_distribution_file.name}")
+ if "-sources.tar.gz" not in sdist_distribution_file.name:
+ # no need to delete - we are in temporary container
+ tmpdir = mkdtemp()
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-m",
+ "pip",
+ "wheel",
+ "--wheel-dir",
+ tmpdir,
+ "--no-deps",
+ "--no-cache",
+ "--no-binary",
+ ":all:",
+ sdist_distribution_file.as_posix(),
+ ],
+ check=False,
+ )
+ if result.returncode != 0:
+ console.print(f"[red]Error installing
{sdist_distribution_file.name}")
+ sys.exit(result.returncode)
+ console.print(
+ f"[green]Sdist package {sdist_distribution_file.name}
can be built into wheels"
+ )
+ console.print("[green]Sdist package is installed
successfully.")
console.print("[green]Airflow packages built successfully")
@@ -147,6 +151,6 @@ mark_git_directory_as_safe()
build_airflow_packages(DISTRIBUTION_FORMAT)
-for file in (AIRFLOW_ROOT_PATH / "dist").glob("apache*"):
+for file in AIRFLOW_DIST_PATH.glob("apache_airflow*"):
console.print(file.name)
console.print()
