This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 23fde82f5d79882c44d154bcc5a8ae19bfd8d01e Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Dec 11 16:11:35 2023 +0100 Fix constraints generation when there are no differences (#36168) The new constraints generation implemented in #36158 had a bug that constraint files have not been generated when there was no changes (wrong file has been deleted). This resulted in PROD files generation failing when there were no new releases to one of 670 dependencies. This PR fixes it and improves the diagnostics of comstraints generation: * more information printed about generated files * using shorten method wnen running commands (will add ellipsis if longer) * failing when there are not constraints to upload - indicating a bug in the processs of constraints generation rather than when downloaded constraints are missing. (cherry picked from commit 20a7f929f284cbc1ebda3ecf46ef1174309c1b15) --- .github/actions/build-ci-images/action.yml | 1 + .github/workflows/ci.yml | 1 + .../airflow_breeze/commands/release_management_commands.py | 11 +++++++++++ scripts/in_container/in_container_utils.py | 3 ++- scripts/in_container/run_generate_constraints.py | 14 +++++++++++--- 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/actions/build-ci-images/action.yml b/.github/actions/build-ci-images/action.yml index f950c53b4e..c437835a4d 100644 --- a/.github/actions/build-ci-images/action.yml +++ b/.github/actions/build-ci-images/action.yml @@ -45,3 +45,4 @@ runs: name: source-constraints path: ./files/constraints-*/constraints-*.txt retention-days: 7 + if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7289968c58..1b500e4745 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -536,6 +536,7 @@ jobs: name: constraints path: ./files/constraints-*/constraints-*.txt retention-days: 7 + if-no-files-found: error static-checks: diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 5af44bcc21..b0eca736a1 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -613,6 +613,7 @@ def run_generate_constraints( command="/opt/airflow/scripts/in_container/run_generate_constraints.py", output=output, ) + return ( result.returncode, f"Constraints {shell_params.airflow_constraints_mode}:{shell_params.python}", @@ -624,6 +625,15 @@ CONSTRAINT_PROGRESS_MATCHER = ( ) +def list_generated_constraints(output: Output | None): + get_console(output=output).print("\n[info]List of generated files in './files' folder:[/]\n") + found_files = Path("./files").rglob("*") + for file in sorted(found_files): + if file.is_file(): + get_console(output=output).print(file.as_posix()) + get_console(output=output).print() + + def run_generate_constraints_in_parallel( shell_params_list: list[ShellParams], python_version_list: list[str], @@ -764,6 +774,7 @@ def generate_constraints( if return_code != 0: get_console().print(f"[error]There was an error when generating constraints: {info}[/]") sys.exit(return_code) + list_generated_constraints(output=None) SDIST_FILENAME_PREFIX = "apache-airflow-providers-" diff --git a/scripts/in_container/in_container_utils.py b/scripts/in_container/in_container_utils.py index 459123e34b..6ba35195df 100644 --- a/scripts/in_container/in_container_utils.py +++ b/scripts/in_container/in_container_utils.py @@ -18,6 +18,7 @@ from __future__ import annotations import shlex import subprocess +import textwrap from contextlib import contextmanager import rich_click as click @@ -30,7 +31,7 @@ console = Console(width=400, color_system="standard") @contextmanager def ci_group(group_name: str, github_actions: bool): if github_actions: - console.print(f"::group::{group_name[:200]}[/]", markup=False) + console.print(f"::group::{textwrap.shorten(group_name, width=200)}", markup=False) console.print(group_name, markup=False) try: yield diff --git a/scripts/in_container/run_generate_constraints.py b/scripts/in_container/run_generate_constraints.py index 65cd41b206..e03eff5126 100755 --- a/scripts/in_container/run_generate_constraints.py +++ b/scripts/in_container/run_generate_constraints.py @@ -154,6 +154,7 @@ def freeze_packages_to_file(config_params: ConfigParams, file: TextIO) -> None: check=True, capture_output=True, ) + count_lines = 0 for line in sorted(result.stdout.split("\n")): if line.startswith(("apache_airflow", "apache-airflow==", "/opt/airflow", "#", "-e")): continue @@ -161,9 +162,11 @@ def freeze_packages_to_file(config_params: ConfigParams, file: TextIO) -> None: continue if line.strip() == "": continue + count_lines += 1 file.write(line) file.write("\n") - console.print(f"[green]Constraints generated to file: {file.name}") + file.flush() + console.print(f"[green]Constraints generated to file: {file.name}. Wrote {count_lines} lines") def download_latest_constraint_file(config_params: ConfigParams): @@ -201,8 +204,8 @@ def diff_constraints(config_params: ConfigParams) -> None: ) if result.returncode == 0: console.print("[green]No changes in constraints files. exiting") - config_params.current_constraints_file.unlink(missing_ok=True) - sys.exit(0) + config_params.constraints_diff_file.unlink(missing_ok=True) + return result = run_command( [ "diff", @@ -444,6 +447,11 @@ def generate_constraints( else: console.print(f"[red]Unknown constraints mode: {airflow_constraints_mode}") sys.exit(1) + console.print("[green]Generated constraints:") + files = config_params.constraints_dir.rglob("*.txt") + for file in files: + console.print(file.as_posix()) + console.print() if __name__ == "__main__":
