This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ada9f15be8 Remove generated provider_packages dir (#36138)
ada9f15be8 is described below
commit ada9f15be8b4ed2f76562d30c63dc0c9d08d4fb1
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Dec 9 10:40:00 2023 +0100
Remove generated provider_packages dir (#36138)
When release manager generates provider packages, sources for them
are temporarily copied to subfolders of `dist\provider_packages`.
Generally we remove all such code (unless
`--skip-deleting-generated-files` flag is used) - and we do it
for the sources but we do not delete `provider_packages` folder and
subfolders coming from packages in `apache` and few other nested
provider packages. This causes some warnings when packages are
being signed as the provider_packages directory is attempted to
be signed (and fails).
We remove the generated folder now unless the `--skip*` flag is used.
---
dev/breeze/src/airflow_breeze/commands/release_management_commands.py | 3 +++
dev/breeze/src/airflow_breeze/global_constants.py | 1 +
dev/breeze/src/airflow_breeze/utils/packages.py | 4 ++--
dev/breeze/src/airflow_breeze/utils/path_utils.py | 1 +
4 files changed, 7 insertions(+), 2 deletions(-)
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index d0a837e88b..ad3df865f5 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -132,6 +132,7 @@ from airflow_breeze.utils.path_utils import (
AIRFLOW_WWW_DIR,
CONSTRAINTS_CACHE_DIR,
DIST_DIR,
+ GENERATED_PROVIDER_PACKAGES_DIR,
PROVIDER_METADATA_JSON_FILE_PATH,
cleanup_python_generated_files,
)
@@ -600,6 +601,8 @@ def prepare_provider_packages(
else:
get_console().print(f"\n[success]Generated package
[special]{provider_id}")
success_packages.append(provider_id)
+ if not skip_deleting_generated_files:
+ shutil.rmtree(GENERATED_PROVIDER_PACKAGES_DIR, ignore_errors=True)
get_console().print()
get_console().print("\n[info]Summary of prepared packages:\n")
provider_action_summary("Success", MessageType.SUCCESS, success_packages)
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 25227984ad..6d8306058d 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -82,6 +82,7 @@ ALLOWED_EXECUTORS = [
"CeleryKubernetesExecutor",
"SequentialExecutor",
]
+
DEFAULT_ALLOWED_EXECUTOR = ALLOWED_EXECUTORS[0]
START_AIRFLOW_ALLOWED_EXECUTORS = ["LocalExecutor", "CeleryExecutor",
"SequentialExecutor"]
START_AIRFLOW_DEFAULT_ALLOWED_EXECUTOR = START_AIRFLOW_ALLOWED_EXECUTORS[0]
diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py
b/dev/breeze/src/airflow_breeze/utils/packages.py
index 968edf2298..8e734d3d40 100644
--- a/dev/breeze/src/airflow_breeze/utils/packages.py
+++ b/dev/breeze/src/airflow_breeze/utils/packages.py
@@ -37,8 +37,8 @@ from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.path_utils import (
AIRFLOW_PROVIDERS_ROOT,
BREEZE_SOURCES_ROOT,
- DIST_DIR,
DOCS_ROOT,
+ GENERATED_PROVIDER_PACKAGES_DIR,
PROVIDER_DEPENDENCIES_JSON_FILE_PATH,
)
from airflow_breeze.utils.publish_docs_helpers import (
@@ -347,7 +347,7 @@ def get_documentation_package_path(provider_id: str) ->
Path:
def get_target_root_for_copied_provider_sources(provider_id: str) -> Path:
- return (DIST_DIR / "provider_packages").joinpath(*provider_id.split("."))
+ return GENERATED_PROVIDER_PACKAGES_DIR.joinpath(*provider_id.split("."))
def get_pip_package_name(provider_id: str) -> str:
diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py
b/dev/breeze/src/airflow_breeze/utils/path_utils.py
index 426c95e98b..cc2e302e4d 100644
--- a/dev/breeze/src/airflow_breeze/utils/path_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py
@@ -288,6 +288,7 @@ HOOKS_DIR = AIRFLOW_SOURCES_ROOT / "hooks"
KUBE_DIR = AIRFLOW_SOURCES_ROOT / ".kube"
LOGS_DIR = AIRFLOW_SOURCES_ROOT / "logs"
DIST_DIR = AIRFLOW_SOURCES_ROOT / "dist"
+GENERATED_PROVIDER_PACKAGES_DIR = DIST_DIR / "provider_packages"
DOCS_DIR = AIRFLOW_SOURCES_ROOT / "docs"
SCRIPTS_CI_DIR = AIRFLOW_SOURCES_ROOT / "scripts" / "ci"
SCRIPTS_CI_DOCKER_COMPOSE_DIR = SCRIPTS_CI_DIR / "docker-compose"
