This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v2-10-test by this push:
new 66b44396301 Fix edge-case when conflicting constraints prevent k8s env
creation (… (#43298)
66b44396301 is described below
commit 66b443963014081c04321fcf2c19c11d8e1ee22f
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Oct 23 14:58:22 2024 +0200
Fix edge-case when conflicting constraints prevent k8s env creation (…
(#43298)
* Fix edge-case when conflicting constraints prevent k8s env creation
(#43276)
The #36930 added constraints to creation of k8s environment, but it
had a side effect - the constraints could not be created if source
of airflow had dependencies conflicting with constraints (which
might happen for example when we upgrade FAB - because locally
pinned FAB might be different than the one in constraints).
Also the constraints were "hard-coded" - the python version,
branch and github repository were hard-coded.
This PR fixes both problems:
* constraints URL is dynamically generated based on current
branch, repo and python version
* in case attempts to create the venv with constraints fails,
we attempt to install it again without constraints
(cherry picked from commit 274b6e1168f84561e8f652a1a4a6fc82aeadc477)
* Update k8s_requirements.txt
---
.../src/airflow_breeze/utils/kubernetes_utils.py | 38 +++++++++++++++++++---
scripts/ci/kubernetes/k8s_requirements.txt | 2 +-
2 files changed, 34 insertions(+), 6 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
index 34c9db0766e..69703b4692b 100644
--- a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
@@ -33,9 +33,11 @@ from time import sleep
from typing import Any, NamedTuple
from urllib import request
+from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
from airflow_breeze.global_constants import (
ALLOWED_ARCHITECTURES,
ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS,
+ APACHE_AIRFLOW_GITHUB_REPOSITORY,
HELM_VERSION,
KIND_VERSION,
PIP_VERSION,
@@ -299,7 +301,7 @@ def _requirements_changed() -> bool:
def _install_packages_in_k8s_virtualenv():
- install_command = [
+ install_command_no_constraints = [
str(PYTHON_BIN_PATH),
"-m",
"pip",
@@ -311,16 +313,42 @@ def _install_packages_in_k8s_virtualenv():
capture_output = True
if get_verbose():
capture_output = False
+ python_major_minor_version = run_command(
+ [
+ str(PYTHON_BIN_PATH),
+ "-c",
+ "import sys;
print(f'{sys.version_info.major}.{sys.version_info.minor}')",
+ ],
+ capture_output=True,
+ check=True,
+ text=True,
+ ).stdout.strip()
+ install_command_with_constraints = install_command_no_constraints.copy()
+ install_command_with_constraints.extend(
+ [
+ "--constraint",
+ "https://raw.githubusercontent.com/";
+ f"{APACHE_AIRFLOW_GITHUB_REPOSITORY}/"
+
f"constraints-{AIRFLOW_BRANCH}/constraints-{python_major_minor_version}.txt",
+ ],
+ )
install_packages_result = run_command(
- install_command, check=False, capture_output=capture_output,
text=True, env=env
+ install_command_with_constraints, check=False,
capture_output=capture_output, text=True, env=env
)
if install_packages_result.returncode != 0:
- get_console().print(
- f"[error]Error when installing packages from :
{K8S_REQUIREMENTS_PATH.resolve()}[/]\n"
- )
if not get_verbose():
get_console().print(install_packages_result.stdout)
get_console().print(install_packages_result.stderr)
+ install_packages_result = run_command(
+ install_command_no_constraints, check=False,
capture_output=capture_output, text=True, env=env
+ )
+ if install_packages_result.returncode != 0:
+ get_console().print(
+ f"[error]Error when installing packages from :
{K8S_REQUIREMENTS_PATH.resolve()}[/]\n"
+ )
+ if not get_verbose():
+ get_console().print(install_packages_result.stdout)
+ get_console().print(install_packages_result.stderr)
return install_packages_result
diff --git a/scripts/ci/kubernetes/k8s_requirements.txt
b/scripts/ci/kubernetes/k8s_requirements.txt
index ebef4fa0f44..8642ea87602 100644
--- a/scripts/ci/kubernetes/k8s_requirements.txt
+++ b/scripts/ci/kubernetes/k8s_requirements.txt
@@ -1 +1 @@
--e .[devel-devscripts,devel-tests,cncf.kubernetes] --constraint
"https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-3.8.txt";
+-e .[devel-devscripts,devel-tests,cncf.kubernetes]
