This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 0b46c8de3 fix(ci): ensure wheels are built with older manylinux (#2351)
0b46c8de3 is described below
commit 0b46c8de3fb7e3cc1e701d56a749b12e3f6c1009
Author: David Li <[email protected]>
AuthorDate: Wed Dec 4 20:30:45 2024 -0500
fix(ci): ensure wheels are built with older manylinux (#2351)
- Build binaries separately using an older glibc.
- Use a separate image with a new docker to run cibuildwheel.
Fixes #2350.
---
.env | 2 +-
.github/workflows/packaging.yml | 10 +++-
.../python-wheel-manylinux-relocate.dockerfile | 28 +++++++++
ci/scripts/python_util.sh | 30 ++++++----
ci/scripts/python_wheel_unix_build.sh | 69 ++++------------------
...unix_build.sh => python_wheel_unix_relocate.sh} | 37 +-----------
docker-compose.yml | 22 ++++++-
7 files changed, 89 insertions(+), 109 deletions(-)
diff --git a/.env b/.env
index 068727961..9e5ff8fae 100644
--- a/.env
+++ b/.env
@@ -29,7 +29,7 @@ ARCH_CONDA_FORGE=linux_64_
# Default versions for various dependencies
JDK=11
-MANYLINUX=2-28
+MANYLINUX=2014
MAVEN=3.6.3
PLATFORM=linux/amd64
PYTHON=3.9
diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml
index 2cda890e1..135ee1c99 100644
--- a/.github/workflows/packaging.yml
+++ b/.github/workflows/packaging.yml
@@ -568,7 +568,7 @@ jobs:
is_pr: true
include:
- {arch: amd64, platform: linux/amd64}
- - {arch: arm64v8, platform: linux/arm64}
+ - {arch: arm64v8, platform: linux/arm64/v8}
steps:
- uses: actions/download-artifact@v4
with:
@@ -599,11 +599,16 @@ jobs:
- name: Build wheel
env:
ARCH: ${{ matrix.arch }}
+ PLATFORM: ${{ matrix.platform }}
run: |
pushd adbc
docker compose run \
-e SETUPTOOLS_SCM_PRETEND_VERSION=$VERSION \
- python-wheel-manylinux
+ python-wheel-manylinux-build
+
+ docker compose run \
+ -e SETUPTOOLS_SCM_PRETEND_VERSION=$VERSION \
+ python-wheel-manylinux-relocate
popd
- name: Archive wheels
@@ -742,6 +747,7 @@ jobs:
$PYTHON -m venv build-env
source build-env/bin/activate
./ci/scripts/python_wheel_unix_build.sh $ARCH $(pwd) $(pwd)/build
+ ./ci/scripts/python_wheel_unix_relocate.sh $ARCH $(pwd) $(pwd)/build
popd
- name: Archive wheels
diff --git a/ci/docker/python-wheel-manylinux-relocate.dockerfile
b/ci/docker/python-wheel-manylinux-relocate.dockerfile
new file mode 100644
index 000000000..c25d7457f
--- /dev/null
+++ b/ci/docker/python-wheel-manylinux-relocate.dockerfile
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM debian:bookworm-slim
+
+RUN apt update \
+ && apt install -y \
+ docker.io \
+ git \
+ patchelf \
+ python-is-python3 \
+ python3-full \
+ python3-pip \
+ && apt clean
diff --git a/ci/scripts/python_util.sh b/ci/scripts/python_util.sh
index 5be89e798..abeabd78b 100644
--- a/ci/scripts/python_util.sh
+++ b/ci/scripts/python_util.sh
@@ -21,6 +21,24 @@ set -ex
COMPONENTS="adbc_driver_bigquery adbc_driver_manager adbc_driver_flightsql
adbc_driver_postgresql adbc_driver_sqlite adbc_driver_snowflake"
+function find_drivers {
+ local -r build_dir="${1}/${VCPKG_ARCH}"
+
+ if [[ $(uname) == "Linux" ]]; then
+ export
ADBC_BIGQUERY_LIBRARY=${build_dir}/lib/libadbc_driver_bigquery.so
+ export
ADBC_FLIGHTSQL_LIBRARY=${build_dir}/lib/libadbc_driver_flightsql.so
+ export
ADBC_POSTGRESQL_LIBRARY=${build_dir}/lib/libadbc_driver_postgresql.so
+ export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.so
+ export
ADBC_SNOWFLAKE_LIBRARY=${build_dir}/lib/libadbc_driver_snowflake.so
+ else # macOS
+ export
ADBC_BIGQUERY_LIBRARY=${build_dir}/lib/libadbc_driver_bigquery.dylib
+ export
ADBC_FLIGHTSQL_LIBRARY=${build_dir}/lib/libadbc_driver_flightsql.dylib
+ export
ADBC_POSTGRESQL_LIBRARY=${build_dir}/lib/libadbc_driver_postgresql.dylib
+ export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.dylib
+ export
ADBC_SNOWFLAKE_LIBRARY=${build_dir}/lib/libadbc_driver_snowflake.dylib
+ fi
+}
+
function build_drivers {
local -r source_dir="$1"
local -r build_dir="$2/${VCPKG_ARCH}"
@@ -35,20 +53,12 @@ function build_drivers {
# Add our custom triplets
export VCPKG_OVERLAY_TRIPLETS="${source_dir}/ci/vcpkg/triplets/"
+ find_drivers "${2}"
+
if [[ $(uname) == "Linux" ]]; then
- export
ADBC_BIGQUERY_LIBRARY=${build_dir}/lib/libadbc_driver_bigquery.so
- export
ADBC_FLIGHTSQL_LIBRARY=${build_dir}/lib/libadbc_driver_flightsql.so
- export
ADBC_POSTGRESQL_LIBRARY=${build_dir}/lib/libadbc_driver_postgresql.so
- export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.so
- export
ADBC_SNOWFLAKE_LIBRARY=${build_dir}/lib/libadbc_driver_snowflake.so
export VCPKG_DEFAULT_TRIPLET="${VCPKG_ARCH}-linux-static-release"
export CMAKE_ARGUMENTS=""
else # macOS
- export
ADBC_BIGQUERY_LIBRARY=${build_dir}/lib/libadbc_driver_bigquery.dylib
- export
ADBC_FLIGHTSQL_LIBRARY=${build_dir}/lib/libadbc_driver_flightsql.dylib
- export
ADBC_POSTGRESQL_LIBRARY=${build_dir}/lib/libadbc_driver_postgresql.dylib
- export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.dylib
- export
ADBC_SNOWFLAKE_LIBRARY=${build_dir}/lib/libadbc_driver_snowflake.dylib
export VCPKG_DEFAULT_TRIPLET="${VCPKG_ARCH}-osx-static-release"
if [[ "${VCPKG_ARCH}" = "x64" ]]; then
export CMAKE_ARGUMENTS="-DCMAKE_OSX_ARCHITECTURES=x86_64"
diff --git a/ci/scripts/python_wheel_unix_build.sh
b/ci/scripts/python_wheel_unix_build.sh
index 693e11fff..f0f2b8637 100755
--- a/ci/scripts/python_wheel_unix_build.sh
+++ b/ci/scripts/python_wheel_unix_build.sh
@@ -38,7 +38,7 @@ function check_visibility {
grep ' T ' nm_arrow.log | grep -v -E
'(Adbc|DriverInit|\b_init\b|\b_fini\b)' | cat - > visible_symbols.log
if [[ -f visible_symbols.log && `cat visible_symbols.log | wc -l` -eq 0
]]; then
- return 0
+ echo "No unexpected symbols exported by $1"
else
echo "== Unexpected symbols exported by $1 =="
cat visible_symbols.log
@@ -46,34 +46,22 @@ function check_visibility {
exit 1
fi
-}
-function check_wheels {
- if [[ $(uname) == "Linux" ]]; then
- echo "=== Tag $component wheel with manylinux${MANYLINUX_VERSION} ==="
- auditwheel repair "$@" -L . -w repaired_wheels
- else # macOS
- echo "=== Tag $component wheel with macOS ==="
- delocate-wheel -v -k -w repaired_wheels "$@"
+ # Also check the max glibc version, to avoid accidentally bumping our
+ # manylinux requirement
+ local -r glibc_max=2.17
+ local -r glibc_requirement=$(grep -Eo 'GLIBC_\S+' nm_arrow.log | awk -F_
'{print $2}' | sort --version-sort -u | tail -n1)
+ local -r maxver=$(echo -e "${glibc_requirement}\n${glibc_max}" | sort
--version-sort | tail -n1)
+ if [[ "${maxver}" != "2.17" ]]; then
+ echo "== glibc check failed for $1 =="
+ echo "Expected ${glibc_max} but found ${glibc_requirement}"
+ exit 1
fi
}
echo "=== Set up platform variables ==="
setup_build_vars "${arch}"
-# XXX: when we manually retag the wheel, we have to use the right arch
-# tag accounting for cross-compiling, hence the replacements
-PLAT_NAME=$(python -c "import sysconfig; print(sysconfig.get_platform()\
- .replace('-x86_64', '-${PYTHON_ARCH}')\
- .replace('-arm64', '-${PYTHON_ARCH}')\
- .replace('-universal2', '-${PYTHON_ARCH}'))")
-if [[ "${arch}" = "arm64v8" && "$(uname)" = "Darwin" ]]; then
- # Manually override the tag in this case - CI will naively generate
- # "macosx_10_9_arm64" but this isn't a 'real' tag because the first
- # version of macOS supporting AArch64 was macOS 11 Big Sur
- PLAT_NAME="macosx_11_0_arm64"
-fi
-
echo "=== Building C/C++ driver components ==="
# Sets ADBC_POSTGRESQL_LIBRARY, ADBC_SQLITE_LIBRARY
build_drivers "${source_dir}" "${build_dir}"
@@ -84,40 +72,3 @@ check_visibility $ADBC_FLIGHTSQL_LIBRARY
check_visibility $ADBC_POSTGRESQL_LIBRARY
check_visibility $ADBC_SQLITE_LIBRARY
check_visibility $ADBC_SNOWFLAKE_LIBRARY
-
-# https://github.com/pypa/pip/issues/7555
-# Get the latest pip so we have in-tree-build by default
-python -m pip install --upgrade pip auditwheel 'cibuildwheel>=2.21.2' delocate
setuptools wheel
-
-# Build with Cython debug info
-export ADBC_BUILD_TYPE="debug"
-
-for component in $COMPONENTS; do
- pushd ${source_dir}/python/$component
-
- echo "=== Clean build artifacts ==="
- rm -rf ./build ./dist ./repaired_wheels ./$component/*.so
./$component/*.so.*
-
- echo "=== Check $component version ==="
- python $component/_version.py
-
- echo "=== Building $component wheel ==="
- # First, create an sdist, which 1) bundles the C++ sources and 2)
- # embeds the git tag. cibuildwheel may copy into a Docker
- # container during build, but it only copies the package
- # directory, which omits the C++ sources and .git directory,
- # causing the build to fail.
- python setup.py sdist
- if [[ "$component" = "adbc_driver_manager" ]]; then
- python -m cibuildwheel --output-dir repaired_wheels/
dist/$component-*.tar.gz
- else
- python -m pip wheel --no-deps -w dist -vvv .
-
- # Retag the wheel
- python "${script_dir}/python_wheel_fix_tag.py"
--plat-name="${PLAT_NAME}" dist/$component-*.whl
-
- check_wheels dist/$component-*.whl
- fi
-
- popd
-done
diff --git a/ci/scripts/python_wheel_unix_build.sh
b/ci/scripts/python_wheel_unix_relocate.sh
similarity index 73%
copy from ci/scripts/python_wheel_unix_build.sh
copy to ci/scripts/python_wheel_unix_relocate.sh
index 693e11fff..dde802aee 100755
--- a/ci/scripts/python_wheel_unix_build.sh
+++ b/ci/scripts/python_wheel_unix_relocate.sh
@@ -26,32 +26,10 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd
)"
source "${script_dir}/python_util.sh"
-function check_visibility {
- if [[ $(uname) != "Linux" ]]; then
- return 0
- fi
- nm --demangle --dynamic $1 > nm_arrow.log
-
- # Filter out Arrow symbols and see if anything remains.
- # '_init' and '_fini' symbols may or not be present, we don't care.
- # (note we must ignore the grep exit status when no match is found)
- grep ' T ' nm_arrow.log | grep -v -E
'(Adbc|DriverInit|\b_init\b|\b_fini\b)' | cat - > visible_symbols.log
-
- if [[ -f visible_symbols.log && `cat visible_symbols.log | wc -l` -eq 0
]]; then
- return 0
- else
- echo "== Unexpected symbols exported by $1 =="
- cat visible_symbols.log
- echo "================================================"
-
- exit 1
- fi
-}
-
function check_wheels {
if [[ $(uname) == "Linux" ]]; then
echo "=== Tag $component wheel with manylinux${MANYLINUX_VERSION} ==="
- auditwheel repair "$@" -L . -w repaired_wheels
+ auditwheel repair "$@" -L . -w repaired_wheels --plat
manylinux_2_17_${CIBW_ARCHS}
else # macOS
echo "=== Tag $component wheel with macOS ==="
delocate-wheel -v -k -w repaired_wheels "$@"
@@ -60,6 +38,7 @@ function check_wheels {
echo "=== Set up platform variables ==="
setup_build_vars "${arch}"
+find_drivers "${build_dir}"
# XXX: when we manually retag the wheel, we have to use the right arch
# tag accounting for cross-compiling, hence the replacements
@@ -74,17 +53,7 @@ if [[ "${arch}" = "arm64v8" && "$(uname)" = "Darwin" ]]; then
PLAT_NAME="macosx_11_0_arm64"
fi
-echo "=== Building C/C++ driver components ==="
-# Sets ADBC_POSTGRESQL_LIBRARY, ADBC_SQLITE_LIBRARY
-build_drivers "${source_dir}" "${build_dir}"
-
-# Check that we don't expose any unwanted symbols
-check_visibility $ADBC_BIGQUERY_LIBRARY
-check_visibility $ADBC_FLIGHTSQL_LIBRARY
-check_visibility $ADBC_POSTGRESQL_LIBRARY
-check_visibility $ADBC_SQLITE_LIBRARY
-check_visibility $ADBC_SNOWFLAKE_LIBRARY
-
+echo "=== Relocating wheels ==="
# https://github.com/pypa/pip/issues/7555
# Get the latest pip so we have in-tree-build by default
python -m pip install --upgrade pip auditwheel 'cibuildwheel>=2.21.2' delocate
setuptools wheel
diff --git a/docker-compose.yml b/docker-compose.yml
index a5db95e15..bf961cdb8 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -121,7 +121,10 @@ services:
############################ Python wheels ##################################
- python-wheel-manylinux:
+ # We build on a different image to use an older base image/glibc, then
+ # relocate on a separate image so that we can use a newer docker for
cibuildwheel
+
+ python-wheel-manylinux-build:
image:
${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}-adbc
build:
context: .
@@ -135,11 +138,24 @@ services:
PYTHON: ${PYTHON}
REPO: ${REPO}
VCPKG: ${VCPKG}
+ volumes:
+ - .:/adbc
+ # Must set safe.directory so go/miniver won't error when calling git
+ command: "'git config --global --add safe.directory /adbc &&
/adbc/ci/scripts/python_wheel_unix_build.sh ${ARCH} /adbc /adbc/build'"
+
+ python-wheel-manylinux-relocate:
+ image: ${REPO}:adbc-python-${PYTHON}-wheel-relocate
+ platform: ${PLATFORM}
+ build:
+ context: .
+ cache_from:
+ - ${REPO}:adbc-python-${PYTHON}-wheel-relocate
+ dockerfile: ci/docker/python-wheel-manylinux-relocate.dockerfile
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- .:/adbc
- # Must set safe.directory so miniver won't error when calling git
- command: "'git config --global --add safe.directory /adbc && git config
--global --get safe.directory && /adbc/ci/scripts/python_wheel_unix_build.sh
${ARCH} /adbc /adbc/build'"
+ # Must set safe.directory so go/miniver won't error when calling git
+ command: "bash -c 'git config --global --add safe.directory /adbc &&
python -m venv /venv && source /venv/bin/activate &&
/adbc/ci/scripts/python_wheel_unix_relocate.sh ${ARCH} /adbc /adbc/build'"
python-wheel-manylinux-test:
image: ${ARCH}/python:${PYTHON}-slim
