This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 274b6e1168f Fix edge-case when conflicting constraints prevent k8s env
creation (#43276)
274b6e1168f is described below
commit 274b6e1168f84561e8f652a1a4a6fc82aeadc477
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Oct 23 09:54:26 2024 +0200
Fix edge-case when conflicting constraints prevent k8s env creation (#43276)
The #36930 added constraints to creation of k8s environment, but it
had a side effect - the constraints could not be created if source
of airflow had dependencies conflicting with constraints (which
might happen for example when we upgrade FAB - because locally
pinned FAB might be different than the one in constraints).
Also the constraints were "hard-coded" - the python version,
branch and github repository were hard-coded.
This PR fixes both problems:
* constraints URL is dynamically generated based on current
branch, repo and python version
* in case attempts to create the venv with constraints fails,
we attempt to install it again without constraints
---
.../src/airflow_breeze/utils/kubernetes_utils.py | 38 +++++++++++++++++++---
scripts/ci/kubernetes/k8s_requirements.txt | 1 -
2 files changed, 33 insertions(+), 6 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
index 34c9db0766e..69703b4692b 100644
--- a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
@@ -33,9 +33,11 @@ from time import sleep
from typing import Any, NamedTuple
from urllib import request
+from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
from airflow_breeze.global_constants import (
ALLOWED_ARCHITECTURES,
ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS,
+ APACHE_AIRFLOW_GITHUB_REPOSITORY,
HELM_VERSION,
KIND_VERSION,
PIP_VERSION,
@@ -299,7 +301,7 @@ def _requirements_changed() -> bool:
def _install_packages_in_k8s_virtualenv():
- install_command = [
+ install_command_no_constraints = [
str(PYTHON_BIN_PATH),
"-m",
"pip",
@@ -311,16 +313,42 @@ def _install_packages_in_k8s_virtualenv():
capture_output = True
if get_verbose():
capture_output = False
+ python_major_minor_version = run_command(
+ [
+ str(PYTHON_BIN_PATH),
+ "-c",
+ "import sys;
print(f'{sys.version_info.major}.{sys.version_info.minor}')",
+ ],
+ capture_output=True,
+ check=True,
+ text=True,
+ ).stdout.strip()
+ install_command_with_constraints = install_command_no_constraints.copy()
+ install_command_with_constraints.extend(
+ [
+ "--constraint",
+ "https://raw.githubusercontent.com/";
+ f"{APACHE_AIRFLOW_GITHUB_REPOSITORY}/"
+
f"constraints-{AIRFLOW_BRANCH}/constraints-{python_major_minor_version}.txt",
+ ],
+ )
install_packages_result = run_command(
- install_command, check=False, capture_output=capture_output,
text=True, env=env
+ install_command_with_constraints, check=False,
capture_output=capture_output, text=True, env=env
)
if install_packages_result.returncode != 0:
- get_console().print(
- f"[error]Error when installing packages from :
{K8S_REQUIREMENTS_PATH.resolve()}[/]\n"
- )
if not get_verbose():
get_console().print(install_packages_result.stdout)
get_console().print(install_packages_result.stderr)
+ install_packages_result = run_command(
+ install_command_no_constraints, check=False,
capture_output=capture_output, text=True, env=env
+ )
+ if install_packages_result.returncode != 0:
+ get_console().print(
+ f"[error]Error when installing packages from :
{K8S_REQUIREMENTS_PATH.resolve()}[/]\n"
+ )
+ if not get_verbose():
+ get_console().print(install_packages_result.stdout)
+ get_console().print(install_packages_result.stderr)
return install_packages_result
diff --git a/scripts/ci/kubernetes/k8s_requirements.txt
b/scripts/ci/kubernetes/k8s_requirements.txt
index e04ef564127..2d00510e382 100644
--- a/scripts/ci/kubernetes/k8s_requirements.txt
+++ b/scripts/ci/kubernetes/k8s_requirements.txt
@@ -1,4 +1,3 @@
---constraint
https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.9.txt
-e .[devel-devscripts,devel-tests,cncf.kubernetes]
-e ./providers
-e ./task_sdk
