This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 09bbfb5a955 Fix Python 3.11 work from sources in CI image (#55120)
09bbfb5a955 is described below
commit 09bbfb5a9556bcd0b5faa8066b3d2d2b201fb3ff
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Sep 1 05:14:03 2025 +0200
Fix Python 3.11 work from sources in CI image (#55120)
The #53770 changed the way how we build our Python image - we build
python from released packages. We are building python - in order
to be able to copy it between build and main images in PROD image
in a separate /usr/python/ path and then we symbolically link it
from /usr/local/lib for cpmpatibility with the original Python
base images we used before.
Python in /usr/python is the first on the path so Python shoudl
be used from there by default.
However in Python 3.11 (and only in Python 3.11) when hatch build
backend build the package, apparently python is used from /usr/local/lib
and there (even if LD_LIBRARY_PATH is properly set), python 3.11
run from /usr/local/lib, even if it is a symlink to the /usr/python/bin,
expects to find dynamically linked python libraries in /usr/local/lib.
We also rmeove LD_LIBRARY_PATH that was set before to handle it (but
not completely). Having symbolic links defined for lib entries in both
places is a more complete solution with less side effects.
---
Dockerfile | 19 ++++++++++++-------
Dockerfile.ci | 15 +++++++++++----
scripts/docker/install_os_dependencies.sh | 11 ++++++++++-
3 files changed, 33 insertions(+), 12 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 9ab8061593f..a20fcaaebc6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -197,7 +197,6 @@ function link_python() {
# Links in /usr/local/bin are needed for tools that expect python to be
there
# Links in /usr/python/bin are needed for tools that are detecting home of
python installation including
# lib/site-packages. The /usr/python/bin should be first in PATH in order
to help with the last part.
- ldconfig
for dst in pip3 python3 python3-config; do
src="$(echo "${dst}" | tr -d 3)"
echo "Linking ${dst} in /usr/local/bin and /usr/python/bin"
@@ -209,6 +208,16 @@ function link_python() {
fi
done
done
+ for dst in /usr/python/lib/*
+ do
+ src="/usr/local/lib/$(basename "${dst}")"
+ if [[ -e "${src}" ]]; then
+ rm -rf "${src}"
+ fi
+ echo "Linking ${dst} to ${src}"
+ ln -sv "${dst}" "${src}"
+ done
+ ldconfig
}
function install_debian_runtime_dependencies() {
@@ -1558,11 +1567,9 @@ SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit",
"-o", "nounset", "-o", "n
ARG BASE_IMAGE
# Make sure noninteractive debian install is used and language variables set
-# as well as LD_LIBRARY_PATH to /usr/local/lib and /usr/python/lib is set, so
that
-# shared libraries installed there are found
ENV BASE_IMAGE=${BASE_IMAGE} \
DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8
LC_ALL=C.UTF-8 \
- LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
LD_LIBRARY_PATH=/usr/python/lib:/usr/local/lib \
+ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
PIP_CACHE_DIR=/tmp/.cache/pip \
UV_CACHE_DIR=/tmp/.cache/uv
@@ -1794,11 +1801,9 @@ LABEL org.apache.airflow.distro="debian" \
ARG BASE_IMAGE
# Make sure noninteractive debian install is used and language variables set
-# as well as LD_LIBRARY_PATH to /usr/local/lib and /usr/python/lib is set, so
that
-# shared libraries installed there are found
ENV BASE_IMAGE=${BASE_IMAGE} \
DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8
LC_ALL=C.UTF-8 \
- LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
LD_LIBRARY_PATH=/usr/python/lib:/usr/local/lib \
+ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
PIP_CACHE_DIR=/tmp/.cache/pip \
UV_CACHE_DIR=/tmp/.cache/uv
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 6ae2b7f5eda..77776120374 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -137,7 +137,6 @@ function link_python() {
# Links in /usr/local/bin are needed for tools that expect python to be
there
# Links in /usr/python/bin are needed for tools that are detecting home of
python installation including
# lib/site-packages. The /usr/python/bin should be first in PATH in order
to help with the last part.
- ldconfig
for dst in pip3 python3 python3-config; do
src="$(echo "${dst}" | tr -d 3)"
echo "Linking ${dst} in /usr/local/bin and /usr/python/bin"
@@ -149,6 +148,16 @@ function link_python() {
fi
done
done
+ for dst in /usr/python/lib/*
+ do
+ src="/usr/local/lib/$(basename "${dst}")"
+ if [[ -e "${src}" ]]; then
+ rm -rf "${src}"
+ fi
+ echo "Linking ${dst} to ${src}"
+ ln -sv "${dst}" "${src}"
+ done
+ ldconfig
}
function install_debian_runtime_dependencies() {
@@ -1365,11 +1374,9 @@ ARG
AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow";
ARG DEPENDENCIES_EPOCH_NUMBER="15"
# Make sure noninteractive debian install is used and language variables set
-# As well as LD_LIBRARY_PATH to /usr/local/lib and /usr/python/lib is set, so
that
-# shared libraries installed there are found
ENV BASE_IMAGE=${BASE_IMAGE} \
DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8
LC_ALL=C.UTF-8 \
- LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
LD_LIBRARY_PATH=/usr/python/lib:/usr/local/lib \
+ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} \
INSTALL_MYSQL_CLIENT="true" \
INSTALL_MSSQL_CLIENT="true" \
diff --git a/scripts/docker/install_os_dependencies.sh
b/scripts/docker/install_os_dependencies.sh
index 3f17bdf5e89..9f341789630 100644
--- a/scripts/docker/install_os_dependencies.sh
+++ b/scripts/docker/install_os_dependencies.sh
@@ -119,7 +119,6 @@ function link_python() {
# Links in /usr/local/bin are needed for tools that expect python to be
there
# Links in /usr/python/bin are needed for tools that are detecting home of
python installation including
# lib/site-packages. The /usr/python/bin should be first in PATH in order
to help with the last part.
- ldconfig
for dst in pip3 python3 python3-config; do
src="$(echo "${dst}" | tr -d 3)"
echo "Linking ${dst} in /usr/local/bin and /usr/python/bin"
@@ -131,6 +130,16 @@ function link_python() {
fi
done
done
+ for dst in /usr/python/lib/*
+ do
+ src="/usr/local/lib/$(basename "${dst}")"
+ if [[ -e "${src}" ]]; then
+ rm -rf "${src}"
+ fi
+ echo "Linking ${dst} to ${src}"
+ ln -sv "${dst}" "${src}"
+ done
+ ldconfig
}
function install_debian_runtime_dependencies() {
