This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-8-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 413b22e480fc6ce01c9f49d7b0363e8efe77c25c Author: Jarek Potiuk <[email protected]> AuthorDate: Sat Dec 9 10:40:00 2023 +0100 Remove generated provider_packages dir (#36138) When release manager generates provider packages, sources for them are temporarily copied to subfolders of `dist\provider_packages`. Generally we remove all such code (unless `--skip-deleting-generated-files` flag is used) - and we do it for the sources but we do not delete `provider_packages` folder and subfolders coming from packages in `apache` and few other nested provider packages. This causes some warnings when packages are being signed as the provider_packages directory is attempted to be signed (and fails). We remove the generated folder now unless the `--skip*` flag is used. (cherry picked from commit ada9f15be8b4ed2f76562d30c63dc0c9d08d4fb1) --- dev/breeze/src/airflow_breeze/commands/release_management_commands.py | 3 +++ dev/breeze/src/airflow_breeze/global_constants.py | 1 + dev/breeze/src/airflow_breeze/utils/packages.py | 4 ++-- dev/breeze/src/airflow_breeze/utils/path_utils.py | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index d0a837e88b..ad3df865f5 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -132,6 +132,7 @@ from airflow_breeze.utils.path_utils import ( AIRFLOW_WWW_DIR, CONSTRAINTS_CACHE_DIR, DIST_DIR, + GENERATED_PROVIDER_PACKAGES_DIR, PROVIDER_METADATA_JSON_FILE_PATH, cleanup_python_generated_files, ) @@ -600,6 +601,8 @@ def prepare_provider_packages( else: get_console().print(f"\n[success]Generated package [special]{provider_id}") success_packages.append(provider_id) + if not skip_deleting_generated_files: + shutil.rmtree(GENERATED_PROVIDER_PACKAGES_DIR, ignore_errors=True) get_console().print() get_console().print("\n[info]Summary of prepared packages:\n") provider_action_summary("Success", MessageType.SUCCESS, success_packages) diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 25227984ad..6d8306058d 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -82,6 +82,7 @@ ALLOWED_EXECUTORS = [ "CeleryKubernetesExecutor", "SequentialExecutor", ] + DEFAULT_ALLOWED_EXECUTOR = ALLOWED_EXECUTORS[0] START_AIRFLOW_ALLOWED_EXECUTORS = ["LocalExecutor", "CeleryExecutor", "SequentialExecutor"] START_AIRFLOW_DEFAULT_ALLOWED_EXECUTOR = START_AIRFLOW_ALLOWED_EXECUTORS[0] diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index 968edf2298..8e734d3d40 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -37,8 +37,8 @@ from airflow_breeze.utils.console import get_console from airflow_breeze.utils.path_utils import ( AIRFLOW_PROVIDERS_ROOT, BREEZE_SOURCES_ROOT, - DIST_DIR, DOCS_ROOT, + GENERATED_PROVIDER_PACKAGES_DIR, PROVIDER_DEPENDENCIES_JSON_FILE_PATH, ) from airflow_breeze.utils.publish_docs_helpers import ( @@ -347,7 +347,7 @@ def get_documentation_package_path(provider_id: str) -> Path: def get_target_root_for_copied_provider_sources(provider_id: str) -> Path: - return (DIST_DIR / "provider_packages").joinpath(*provider_id.split(".")) + return GENERATED_PROVIDER_PACKAGES_DIR.joinpath(*provider_id.split(".")) def get_pip_package_name(provider_id: str) -> str: diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 426c95e98b..cc2e302e4d 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -288,6 +288,7 @@ HOOKS_DIR = AIRFLOW_SOURCES_ROOT / "hooks" KUBE_DIR = AIRFLOW_SOURCES_ROOT / ".kube" LOGS_DIR = AIRFLOW_SOURCES_ROOT / "logs" DIST_DIR = AIRFLOW_SOURCES_ROOT / "dist" +GENERATED_PROVIDER_PACKAGES_DIR = DIST_DIR / "provider_packages" DOCS_DIR = AIRFLOW_SOURCES_ROOT / "docs" SCRIPTS_CI_DIR = AIRFLOW_SOURCES_ROOT / "scripts" / "ci" SCRIPTS_CI_DOCKER_COMPOSE_DIR = SCRIPTS_CI_DIR / "docker-compose"
