This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new af007a5fa1 fIx constraints use in CI image after uv change (#37845)
af007a5fa1 is described below
commit af007a5fa179f8880139b846cdef4ee0f22971f4
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Mar 2 11:23:58 2024 +0100
fIx constraints use in CI image after uv change (#37845)
With the change to switch to uv, we skipped constraints being used
in CI image - in effect all PR were not using constraints, but they
were using not constraint dependencues but lowest-direct
mode of installation so direct dependencies would not be upgraded
in such case, only the transitive ones, so the risk of failure was
anyhow small even if someone released a new, breakong dependency.
The reason is that `uv` currently does not support installing
constraints from URL. We had been silently failing back to the
"no-constraints" way in such case (this is default mode if for any reason
constraint build fail in such case.
It introduced the risk that in case 3rd-party breaking dependency
was released it would also start breaking regular PRs,
not only the "canary" build.
We fix it by downloading constraints locally when they are remote and
using them from there.
While this is being worked on in https://github.com/astral-sh/uv/pull/2081
and likely to land in uv 0.1.14, it's also a good idea to actually
download the constraints and keep them around - this might be handy
if you want to later use constraints to install "golden" set of
dependencies wihtout necessity to build the right URL - you can always
use `${HOME}/constraints.txt`.
This PR fixes it and also changes the fallback mechanism to perform the
lowest-direct upgrade only in case the constraint build fails, rather
than always run the lowest-dirct upgrade even if constraints install
works fine - this will make sure that most PRs are using exactly the
constraint version of the dependencies (at least the version of
constraints that were generated last time when pyproject.toml changed).
---
Dockerfile | 37 ++++++++++++++++------
Dockerfile.ci | 35 ++++++++++++++------
scripts/docker/common.sh | 12 +++++++
scripts/docker/install_airflow.sh | 23 ++++++++------
.../docker/install_from_docker_context_files.sh | 2 +-
5 files changed, 80 insertions(+), 29 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 1e75319200..bf08405e08 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -558,6 +558,18 @@ function common::get_constraints_location() {
python_version=$(python -c 'import sys;
print(f"{sys.version_info.major}.{sys.version_info.minor}")')
AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt"
fi
+
+ if [[ ${AIRFLOW_CONSTRAINTS_LOCATION} =~ http.* ]]; then
+ echo
+ echo "${COLOR_BLUE}Downloading constraints from
${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"
+ echo
+ curl -sSf -o "${HOME}/constraints.txt"
"${AIRFLOW_CONSTRAINTS_LOCATION}"
+ else
+ echo
+ echo "${COLOR_BLUE}Copying constraints from
${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"
+ echo
+ cp "${AIRFLOW_CONSTRAINTS_LOCATION}" "${HOME}/constraints.txt"
+ fi
}
function common::show_packaging_tool_version_and_location() {
@@ -730,7 +742,7 @@ function
install_airflow_and_providers_from_docker_context_files(){
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS}
"${packaging_flags[@]}" \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
- --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" \
+ --constraint "${HOME}/constraints.txt" \
${reinstalling_apache_airflow_package}
${reinstalling_apache_airflow_providers_packages}
set +x
fi
@@ -826,19 +838,24 @@ function install_airflow() {
pip check
else
echo
- echo "${COLOR_BLUE}Installing all packages with constraints and
upgrade if needed${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing all packages with constraints or upgrade
if needed${COLOR_RESET}"
echo
set -x
- ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS}
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
+ # Install all packages with constraints
+ if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS}
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
\
- --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
- common::install_packaging_tool
- # then upgrade if needed without using constraints to account for new
limits in pyproject.toml
- ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
${RESOLUTION_LOWEST_DIRECT_FLAG} \
- ${ADDITIONAL_PIP_INSTALL_FLAGS} \
- ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
-
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
+ --constraint "${HOME}/constraints.txt"; then
+ echo
+ echo "${COLOR_YELLOW}Likely pyproject.toml has new dependencies
conflicting with constraints.${COLOR_RESET}"
+ echo
+ echo "${COLOR_BLUE}Falling back to no-constraints, lowest-direct
resolution installation.${COLOR_RESET}"
+ echo
+ ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
${RESOLUTION_LOWEST_DIRECT_FLAG} \
+ ${ADDITIONAL_PIP_INSTALL_FLAGS} \
+ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
+
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
+ fi
common::install_packaging_tool
set +x
echo
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 15475969a7..a217ba2938 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -516,6 +516,18 @@ function common::get_constraints_location() {
python_version=$(python -c 'import sys;
print(f"{sys.version_info.major}.{sys.version_info.minor}")')
AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt"
fi
+
+ if [[ ${AIRFLOW_CONSTRAINTS_LOCATION} =~ http.* ]]; then
+ echo
+ echo "${COLOR_BLUE}Downloading constraints from
${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"
+ echo
+ curl -sSf -o "${HOME}/constraints.txt"
"${AIRFLOW_CONSTRAINTS_LOCATION}"
+ else
+ echo
+ echo "${COLOR_BLUE}Copying constraints from
${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"
+ echo
+ cp "${AIRFLOW_CONSTRAINTS_LOCATION}" "${HOME}/constraints.txt"
+ fi
}
function common::show_packaging_tool_version_and_location() {
@@ -666,19 +678,24 @@ function install_airflow() {
pip check
else
echo
- echo "${COLOR_BLUE}Installing all packages with constraints and
upgrade if needed${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing all packages with constraints or upgrade
if needed${COLOR_RESET}"
echo
set -x
- ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS}
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
+ # Install all packages with constraints
+ if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS}
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
\
- --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
- common::install_packaging_tool
- # then upgrade if needed without using constraints to account for new
limits in pyproject.toml
- ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
${RESOLUTION_LOWEST_DIRECT_FLAG} \
- ${ADDITIONAL_PIP_INSTALL_FLAGS} \
- ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
-
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
+ --constraint "${HOME}/constraints.txt"; then
+ echo
+ echo "${COLOR_YELLOW}Likely pyproject.toml has new dependencies
conflicting with constraints.${COLOR_RESET}"
+ echo
+ echo "${COLOR_BLUE}Falling back to no-constraints, lowest-direct
resolution installation.${COLOR_RESET}"
+ echo
+ ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
${RESOLUTION_LOWEST_DIRECT_FLAG} \
+ ${ADDITIONAL_PIP_INSTALL_FLAGS} \
+ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
+
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
+ fi
common::install_packaging_tool
set +x
echo
diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh
index b39a139799..08b9f697bb 100644
--- a/scripts/docker/common.sh
+++ b/scripts/docker/common.sh
@@ -95,6 +95,18 @@ function common::get_constraints_location() {
python_version=$(python -c 'import sys;
print(f"{sys.version_info.major}.{sys.version_info.minor}")')
AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt"
fi
+
+ if [[ ${AIRFLOW_CONSTRAINTS_LOCATION} =~ http.* ]]; then
+ echo
+ echo "${COLOR_BLUE}Downloading constraints from
${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"
+ echo
+ curl -sSf -o "${HOME}/constraints.txt"
"${AIRFLOW_CONSTRAINTS_LOCATION}"
+ else
+ echo
+ echo "${COLOR_BLUE}Copying constraints from
${AIRFLOW_CONSTRAINTS_LOCATION} to ${HOME}/constraints.txt ${COLOR_RESET}"
+ echo
+ cp "${AIRFLOW_CONSTRAINTS_LOCATION}" "${HOME}/constraints.txt"
+ fi
}
function common::show_packaging_tool_version_and_location() {
diff --git a/scripts/docker/install_airflow.sh
b/scripts/docker/install_airflow.sh
index e8fb806ac6..4397ed34ad 100644
--- a/scripts/docker/install_airflow.sh
+++ b/scripts/docker/install_airflow.sh
@@ -72,19 +72,24 @@ function install_airflow() {
pip check
else
echo
- echo "${COLOR_BLUE}Installing all packages with constraints and
upgrade if needed${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing all packages with constraints or upgrade
if needed${COLOR_RESET}"
echo
set -x
- ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS}
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
+ # Install all packages with constraints
+ if ! ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS}
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
\
- --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
- common::install_packaging_tool
- # then upgrade if needed without using constraints to account for new
limits in pyproject.toml
- ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
${RESOLUTION_LOWEST_DIRECT_FLAG} \
- ${ADDITIONAL_PIP_INSTALL_FLAGS} \
- ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
-
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
+ --constraint "${HOME}/constraints.txt"; then
+ echo
+ echo "${COLOR_YELLOW}Likely pyproject.toml has new dependencies
conflicting with constraints.${COLOR_RESET}"
+ echo
+ echo "${COLOR_BLUE}Falling back to no-constraints, lowest-direct
resolution installation.${COLOR_RESET}"
+ echo
+ ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade
${RESOLUTION_LOWEST_DIRECT_FLAG} \
+ ${ADDITIONAL_PIP_INSTALL_FLAGS} \
+ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
+
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
+ fi
common::install_packaging_tool
set +x
echo
diff --git a/scripts/docker/install_from_docker_context_files.sh
b/scripts/docker/install_from_docker_context_files.sh
index 6e286b9d9c..3c327ad312 100644
--- a/scripts/docker/install_from_docker_context_files.sh
+++ b/scripts/docker/install_from_docker_context_files.sh
@@ -101,7 +101,7 @@ function
install_airflow_and_providers_from_docker_context_files(){
set -x
${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS}
"${packaging_flags[@]}" \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
- --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" \
+ --constraint "${HOME}/constraints.txt" \
${reinstalling_apache_airflow_package}
${reinstalling_apache_airflow_providers_packages}
set +x
fi
