(airflow) branch main updated: Avoid race condition when generating constraints/installing providers (#36155)

[potiuk]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 643b23448f Avoid race condition when generating constraints/installing 
providers (#36155)
643b23448f is described below

commit 643b23448fd809de4f36cbc40d6b0bb45686dad2
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Dec 10 19:45:22 2023 +0100

    Avoid race condition when generating constraints/installing providers 
(#36155)
    
    The change #36131 when switching to docker-compose based running of
    scripts introduced a subtle race condition when several docker-compose
    runs were running in paralell and used the same project_name. This
    happened for example in case of constraints generation - when
    running two Python constraints generation caused docker network to
    be created two times.
    
    This PR fixes the problem for the cases where such run are run in
    parallel - by giving each of the parallel commands unique project_name.
---
 .../src/airflow_breeze/commands/release_management_commands.py      | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git 
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py 
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 8eb7038694..92c0444364 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -609,7 +609,7 @@ def run_generate_constraints(
 ) -> tuple[int, str]:
     result = execute_command_in_shell(
         shell_params,
-        project_name="constraints",
+        project_name=f"constraints-{shell_params.python.replace('.', '-')}",
         
command="/opt/airflow/scripts/in_container/run_generate_constraints.sh",
     )
     fix_ownership_using_docker()
@@ -812,10 +812,11 @@ def get_all_providers_in_dist(package_format: str, 
install_selected_providers: s
 def _run_command_for_providers(
     shell_params: ShellParams,
     list_of_providers: list[str],
+    index: int,
     output: Output | None,
 ) -> tuple[int, str]:
     shell_params.install_selected_providers = " ".join(list_of_providers)
-    result_command = execute_command_in_shell(shell_params, 
project_name="providers")
+    result_command = execute_command_in_shell(shell_params, 
project_name=f"providers-{index}")
     return result_command.returncode, f"{list_of_providers}"
 
 
@@ -949,6 +950,7 @@ def install_provider_packages(
                             "shell_params": shell_params,
                             "list_of_providers": list_of_providers,
                             "output": outputs[index],
+                            "index": index,
                         },
                     )
                     for index, list_of_providers in enumerate(provider_chunks)