This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new ad2b150e0 build: update vcpkg version (and hence dependencies) (#3542)
ad2b150e0 is described below
commit ad2b150e0b3c94fdff82285c3a6715db232ea0de
Author: David Li <[email protected]>
AuthorDate: Thu Oct 9 18:17:05 2025 +0900
build: update vcpkg version (and hence dependencies) (#3542)
- Update to recent vcpkg
- Remove patches that are no longer necessary
- Use our own Docker image to build wheels instead of relying on
upstream
Closes #3516.
---------
Co-authored-by: Sutou Kouhei <[email protected]>
---
.env | 4 +-
ci/docker/python-wheel-manylinux.dockerfile | 66 +++++++++++++---
ci/scripts/install_vcpkg.sh | 2 +-
ci/scripts/python_util.sh | 22 +-----
...-lz4-CMake-https-github.com-lz4-lz4-issue.patch | 87 ----------------------
ci/vcpkg/0002-Retry-downloads.patch | 41 ----------
ci/vcpkg/0003-Fix-CMake-4-OSX.patch | 46 ------------
compose.yaml | 17 ++---
docs/source/driver/postgresql.rst | 2 +-
docs/source/driver/sqlite.rst | 2 +-
10 files changed, 70 insertions(+), 219 deletions(-)
diff --git a/.env b/.env
index cefca8f13..f62b9c65b 100644
--- a/.env
+++ b/.env
@@ -32,7 +32,7 @@ JDK=21
MANYLINUX=2014
MAVEN=3.9.10
PLATFORM=linux/amd64
-PYTHON=3.9
+PYTHON=3.13
GO=1.24.1
ARROW_MAJOR_VERSION=18
DOTNET=8.0
@@ -41,7 +41,7 @@ DOTNET=8.0
# ci/scripts/install_vcpkg.sh script. Keep in sync with apache/arrow .env.
# When updating, also update the docs, which list the version of libpq/SQLite
# that vcpkg (and hence our wheels) ship
-VCPKG="f7423ee180c4b7f40d43402c2feb3859161ef625" # 2024.06.15 Release
+VCPKG="4334d8b4c8916018600212ab4dd4bbdc343065d1" # 2025.09.17 Release
# These are used to tell tests where to find services for integration testing.
# They are valid if the services are started with the compose config.
diff --git a/ci/docker/python-wheel-manylinux.dockerfile
b/ci/docker/python-wheel-manylinux.dockerfile
index 1402f1a19..26b80cf64 100644
--- a/ci/docker/python-wheel-manylinux.dockerfile
+++ b/ci/docker/python-wheel-manylinux.dockerfile
@@ -15,24 +15,66 @@
# specific language governing permissions and limitations
# under the License.
-ARG ARCH
ARG MANYLINUX
+FROM quay.io/pypa/manylinux${MANYLINUX}:latest
+
+ARG CMAKE=4.1.2
+ARG GO
+ARG NINJA=1.13.1
ARG PYTHON
-ARG REPO
ARG VCPKG
+ARG TARGETPLATFORM
-FROM
${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}
-
-ARG ARCH
-ARG GO
+SHELL ["/bin/bash", "-i", "-c"]
+ENTRYPOINT ["/bin/bash", "-i", "-c"]
+# -------------------- System Dependencies --------------------
+# Some of these dependencies are needed to build things like OpenSSL in vcpkg
+RUN ulimit -n 1024 && yum install -y autoconf curl git flex perl-IPC-Cmd unzip
wget yum-utils zip
# docker is aliased to podman by AlmaLinux, but we want real Docker
# (podman is just too different)
-RUN yum remove -y docker ; yum install -y yum-utils
+RUN ulimit -n 1024 && yum remove -y docker
RUN yum-config-manager --add-repo
https://download.docker.com/linux/centos/docker-ce.repo
-RUN yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin
docker-compose-plugin
-# arm64v8 -> arm64
-RUN wget --no-verbose https://go.dev/dl/go${GO}.linux-${ARCH/v8/}.tar.gz && \
- tar -C /usr/local -xzf go${GO}.linux-${ARCH/v8/}.tar.gz && \
- rm go${GO}.linux-${ARCH/v8/}.tar.gz
+RUN ulimit -n 1024 && yum install -y docker-ce docker-ce-cli containerd.io
docker-buildx-plugin docker-compose-plugin
+
+# -------------------- Python --------------------
+RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON/./}-cp${PYTHON/./}) && \
+ echo "export PATH=$PYTHON_ROOT/bin:\$PATH" >> /etc/profile.d/python.sh
+ENV PATH="/opt/python/cp${PYTHON/./}-cp${PYTHON/./}/bin:${PATH}"
+
+# -------------------- CMake, Go --------------------
+RUN mkdir -p /.cache/go-build \
+ && chmod 777 /.cache/go-build \
+ && git config --global --add safe.directory /adbc \
+ && cp /root/.gitconfig /.gitconfig \
+ && chmod 777 /.gitconfig
+
+RUN if [[ ${TARGETPLATFORM} == "linux/amd64" ]]; then
\
+ export ARCH="amd64" CMAKE_ARCH=x86_64;
\
+ elif [[ ${TARGETPLATFORM} == "linux/arm64" ]]; then
\
+ export ARCH="arm64" CMAKE_ARCH=aarch64;
\
+ else
\
+ echo "Unsupported platform: ${TARGETPLATFORM}";
\
+ exit 1;
\
+ fi &&
\
+ wget --no-verbose -O cmake.tar.gz
\
+
https://github.com/Kitware/CMake/releases/download/v${CMAKE}/cmake-${CMAKE}-linux-${CMAKE_ARCH}.tar.gz
&& \
+ wget --no-verbose -O go.tar.gz
https://go.dev/dl/go${GO}.linux-${ARCH}.tar.gz && \
+ tar -C /usr/local -xzf cmake.tar.gz &&
\
+ tar -C /usr/local -xzf go.tar.gz &&
\
+ rm -f cmake.tar.gz go.tar.gz
+
ENV PATH="/usr/local/go/bin:${PATH}"
+
+# -------------------- Ninja --------------------
+RUN mkdir -p /tmp/ninja &&
\
+ wget --no-verbose -O -
"https://github.com/ninja-build/ninja/archive/v${NINJA}.tar.gz"; | \
+ tar -xzf - --directory /tmp/ninja --strip-components=1 &&
\
+ pushd /tmp/ninja &&
\
+ ./configure.py --bootstrap &&
\
+ mv ninja "/usr/local/bin" &&
\
+ rm -rf /tmp/ninja
+
+# -------------------- vcpkg --------------------
+ADD ci/scripts/install_vcpkg.sh /
+RUN /install_vcpkg.sh /opt/vcpkg ${VCPKG} && rm -f /install_vcpkg.sh
diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh
index 25b812af5..1908fc541 100755
--- a/ci/scripts/install_vcpkg.sh
+++ b/ci/scripts/install_vcpkg.sh
@@ -35,7 +35,7 @@ vcpkg_version=${2:-$default_vcpkg_version}
vcpkg_ports_patch=${3:-$default_vcpkg_ports_patch}
# reduce the fetched data using a shallow clone
-git clone --shallow-since=2024-01-01 https://github.com/microsoft/vcpkg
${vcpkg_destination}
+git clone --shallow-since=2025-09-01 https://github.com/microsoft/vcpkg
${vcpkg_destination}
pushd ${vcpkg_destination}
diff --git a/ci/scripts/python_util.sh b/ci/scripts/python_util.sh
index e133870ba..7fab0abe0 100644
--- a/ci/scripts/python_util.sh
+++ b/ci/scripts/python_util.sh
@@ -70,24 +70,8 @@ function build_drivers {
echo "=== Setup VCPKG ==="
- pushd "${VCPKG_ROOT}"
- # XXX: work around lz4 CMakeLists being out of date
- # https://github.com/lz4/lz4/issues/1550
- patch -N -p1 <
"${source_dir}/ci/vcpkg/0001-Work-around-lz4-CMake-https-github.com-lz4-lz4-issue.patch"
-
- # XXX: make vcpkg retry downloads
https://github.com/microsoft/vcpkg/discussions/20583
- patch -N -p1 < "${source_dir}/ci/vcpkg/0002-Retry-downloads.patch"
-
- # XXX: backport fix for CMake 4 and macOS
- patch -N -p1 < "${source_dir}/ci/vcpkg/0003-Fix-CMake-4-OSX.patch"
- popd
-
# Need to install sqlite3 to make CMake be able to find it below
- "${VCPKG_ROOT}/vcpkg" install sqlite3 \
- --overlay-triplets "${VCPKG_OVERLAY_TRIPLETS}" \
- --triplet "${VCPKG_DEFAULT_TRIPLET}"
-
- "${VCPKG_ROOT}/vcpkg" install libpq \
+ "${VCPKG_ROOT}/vcpkg" install libpq sqlite3 \
--overlay-triplets "${VCPKG_OVERLAY_TRIPLETS}" \
--triplet "${VCPKG_DEFAULT_TRIPLET}"
@@ -95,8 +79,6 @@ function build_drivers {
mkdir -p ${build_dir}
pushd ${build_dir}
cmake \
- -DADBC_BUILD_SHARED=ON \
- -DADBC_BUILD_STATIC=ON \
-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
-DCMAKE_INSTALL_LIBDIR=lib \
-DCMAKE_INSTALL_PREFIX=${build_dir} \
@@ -105,6 +87,8 @@ function build_drivers {
${CMAKE_ARGUMENTS} \
-DVCPKG_OVERLAY_TRIPLETS="${VCPKG_OVERLAY_TRIPLETS}" \
-DVCPKG_TARGET_TRIPLET="${VCPKG_DEFAULT_TRIPLET}" \
+ -DADBC_BUILD_SHARED=ON \
+ -DADBC_BUILD_STATIC=ON \
-DADBC_DRIVER_BIGQUERY=ON \
-DADBC_DRIVER_FLIGHTSQL=ON \
-DADBC_DRIVER_MANAGER=ON \
diff --git
a/ci/vcpkg/0001-Work-around-lz4-CMake-https-github.com-lz4-lz4-issue.patch
b/ci/vcpkg/0001-Work-around-lz4-CMake-https-github.com-lz4-lz4-issue.patch
deleted file mode 100644
index 10140587a..000000000
--- a/ci/vcpkg/0001-Work-around-lz4-CMake-https-github.com-lz4-lz4-issue.patch
+++ /dev/null
@@ -1,87 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-From 3306ca07f4a7a83837fe69d0f5d515c3f825f795 Mon Sep 17 00:00:00 2001
-From: David Li <[email protected]>
-Date: Mon, 15 Sep 2025 15:01:53 +0900
-Subject: [PATCH] Work around lz4 CMake
- (https://github.com/lz4/lz4/issues/1550)
-
----
- ports/lz4/CMakeLists.txt | 2 +-
- ports/lz4/lz4-cmake.diff | 25 +++++++++++++++++++++++++
- ports/lz4/portfile.cmake | 2 ++
- 3 files changed, 28 insertions(+), 1 deletion(-)
- create mode 100644 ports/lz4/lz4-cmake.diff
-
-diff --git a/ports/lz4/CMakeLists.txt b/ports/lz4/CMakeLists.txt
-index a35c8b2d1a..50ffd36954 100644
---- a/ports/lz4/CMakeLists.txt
-+++ b/ports/lz4/CMakeLists.txt
-@@ -1,4 +1,4 @@
--cmake_minimum_required(VERSION 3.0)
-+cmake_minimum_required(VERSION 3.10)
- project(lz4 C)
-
- if(MSVC AND BUILD_SHARED_LIBS)
-diff --git a/ports/lz4/lz4-cmake.diff b/ports/lz4/lz4-cmake.diff
-new file mode 100644
-index 0000000000..7c27cd2890
---- /dev/null
-+++ b/ports/lz4/lz4-cmake.diff
-@@ -0,0 +1,25 @@
-+From 678d65fa825b8613ae1c57d6173440e11c568fef Mon Sep 17 00:00:00 2001
-+From: David Li <[email protected]>
-+Date: Mon, 15 Sep 2025 15:15:34 +0900
-+Subject: [PATCH] Change minimum CMake
-+
-+---
-+ build/cmake/CMakeLists.txt | 2 +-
-+ 1 file changed, 1 insertion(+), 1 deletion(-)
-+
-+diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
-+index eb7007b4..8e9517fd 100644
-+--- a/build/cmake/CMakeLists.txt
-++++ b/build/cmake/CMakeLists.txt
-+@@ -10,7 +10,7 @@
-+ # LZ4's CMake support is maintained by Evan Nemerson; when filing
-+ # bugs please mention @nemequ to make sure I see it.
-+
-+-cmake_minimum_required(VERSION 2.8.12)
-++cmake_minimum_required(VERSION 3.10)
-+
-+ set(LZ4_TOP_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
-+
-+--
-+2.48.1
-+
-diff --git a/ports/lz4/portfile.cmake b/ports/lz4/portfile.cmake
-index 80df95aebc..4c2fa23209 100644
---- a/ports/lz4/portfile.cmake
-+++ b/ports/lz4/portfile.cmake
-@@ -4,6 +4,8 @@ vcpkg_from_github(
- REF v1.9.4
- SHA512
043a9acb2417624019d73db140d83b80f1d7c43a6fd5be839193d68df8fd0b3f610d7ed4d628c2a9184f7cde9a0fd1ba9d075d8251298e3eb4b3a77f52736684
- HEAD_REF dev
-+ PATCHES
-+ lz4-cmake.diff
- )
-
- file(COPY "${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt" DESTINATION
"${SOURCE_PATH}")
---
-2.48.1
-
diff --git a/ci/vcpkg/0002-Retry-downloads.patch
b/ci/vcpkg/0002-Retry-downloads.patch
deleted file mode 100644
index 673b2cc49..000000000
--- a/ci/vcpkg/0002-Retry-downloads.patch
+++ /dev/null
@@ -1,41 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-From 6bac044938a6c9fd4832ec636a98cb684ea6f609 Mon Sep 17 00:00:00 2001
-From: David Li <[email protected]>
-Date: Mon, 5 Dec 2022 14:26:42 -0500
-Subject: [PATCH] Retry downloads
-
----
- scripts/cmake/vcpkg_download_distfile.cmake | 2 ++
- 1 file changed, 2 insertions(+)
-
-diff --git a/scripts/cmake/vcpkg_download_distfile.cmake
b/scripts/cmake/vcpkg_download_distfile.cmake
-index 08ca55deb..a1ba00347 100644
---- a/scripts/cmake/vcpkg_download_distfile.cmake
-+++ b/scripts/cmake/vcpkg_download_distfile.cmake
-@@ -73,6 +73,8 @@ function(z_vcpkg_download_distfile_via_aria)
- debug_message("Download Command: ${ARIA2} ${URL} -o temp/${filename}
-l download-${filename}-detailed.log ${headers_param}")
- vcpkg_execute_in_download_mode(
- COMMAND ${ARIA2} ${URL}
-+ --max-tries=5
-+ --retry-wait=20
- -o temp/${arg_FILENAME}
- -l download-${arg_FILENAME}-detailed.log
- ${headers_param}
---
-2.17.1
diff --git a/ci/vcpkg/0003-Fix-CMake-4-OSX.patch
b/ci/vcpkg/0003-Fix-CMake-4-OSX.patch
deleted file mode 100644
index c9b1de2f3..000000000
--- a/ci/vcpkg/0003-Fix-CMake-4-OSX.patch
+++ /dev/null
@@ -1,46 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-From f6c6f20913f2e97fdf452ee31dc878ba8611353b Mon Sep 17 00:00:00 2001
-From: David Li <[email protected]>
-Date: Tue, 16 Sep 2025 09:38:57 +0900
-Subject: [PATCH] Fix CMake 4/OSX
-
-strega-nil/vcpkg@104314ad43f39af13f566d7e8272c0a1708832f7
-https://github.com/microsoft/vcpkg/issues/44697
----
- scripts/get_cmake_vars/CMakeLists.txt | 4 +++-
- 1 file changed, 3 insertions(+), 1 deletion(-)
-
-diff --git a/scripts/get_cmake_vars/CMakeLists.txt
b/scripts/get_cmake_vars/CMakeLists.txt
-index 934515dd18..a8ccee3ff9 100644
---- a/scripts/get_cmake_vars/CMakeLists.txt
-+++ b/scripts/get_cmake_vars/CMakeLists.txt
-@@ -133,7 +133,9 @@ macro(_vcpkg_adjust_flags flag_var)
- string(APPEND ${flag_var} " -arch ${arch}")
- endforeach()
- endif()
-- string(APPEND ${flag_var} " -isysroot ${CMAKE_OSX_SYSROOT}")
-+ if(CMAKE_OSX_SYSROOT)
-+ string(APPEND ${flag_var} " -isysroot ${CMAKE_OSX_SYSROOT}")
-+ endif()
- if (CMAKE_OSX_DEPLOYMENT_TARGET)
- list(GET VCPKG_LANGUAGES 0 lang)
- string(APPEND ${flag_var} "
${CMAKE_${lang}_OSX_DEPLOYMENT_TARGET_FLAG}${CMAKE_OSX_DEPLOYMENT_TARGET}")
---
-2.48.1
-
diff --git a/compose.yaml b/compose.yaml
index 958e52193..b020c4b9a 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -171,32 +171,31 @@ services:
# relocate on a separate image so that we can use a newer docker for
cibuildwheel
python-wheel-manylinux-build:
- image:
${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}-adbc
+ image:
ghcr.io/apache/arrow-adbc:manylinux${MANYLINUX}-go${GO}-python${PYTHON}-vcpkg${VCPKG}
build:
context: .
- cache_from:
- -
${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}-adbc
dockerfile: ci/docker/python-wheel-manylinux.dockerfile
args:
- ARCH: ${ARCH}
GO: ${GO}
MANYLINUX: ${MANYLINUX}
PYTHON: ${PYTHON}
- REPO: ${REPO}
VCPKG: ${VCPKG}
+ platforms:
+ - linux/amd64
+ - linux/arm64
volumes:
- .:/adbc
# Must set safe.directory so go/miniver won't error when calling git
command: "'git config --global --add safe.directory /adbc &&
/adbc/ci/scripts/python_wheel_unix_build.sh ${ARCH} /adbc /adbc/build'"
python-wheel-manylinux-relocate:
- image: ${REPO}:adbc-python-${PYTHON}-wheel-relocate
- platform: ${PLATFORM}
+ image: ghcr.io/apache/arrow-adbc:python${PYTHON}
build:
context: .
- cache_from:
- - ${REPO}:adbc-python-${PYTHON}-wheel-relocate
dockerfile: ci/docker/python-wheel-manylinux-relocate.dockerfile
+ platforms:
+ - linux/amd64
+ - linux/arm64
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- .:/adbc
diff --git a/docs/source/driver/postgresql.rst
b/docs/source/driver/postgresql.rst
index 0d57c4de1..229dedb98 100644
--- a/docs/source/driver/postgresql.rst
+++ b/docs/source/driver/postgresql.rst
@@ -315,7 +315,7 @@ Software Versions
=================
For Python wheels, the shipped version of the PostgreSQL client libraries is
-15.2. For conda-forge packages, the version of libpq is the same as the
+16.9. For conda-forge packages, the version of libpq is the same as the
version of libpq in your Conda environment.
The PostgreSQL driver is tested against PostgreSQL versions 11 through 16.
diff --git a/docs/source/driver/sqlite.rst b/docs/source/driver/sqlite.rst
index a1c113d3d..5b3c25b65 100644
--- a/docs/source/driver/sqlite.rst
+++ b/docs/source/driver/sqlite.rst
@@ -242,6 +242,6 @@ Driver-specific options:
Software Versions
=================
-For Python wheels, the shipped version of SQLite is 3.40.1. For conda-forge
+For Python wheels, the shipped version of SQLite is 3.50.4. For conda-forge
packages, the version of sqlite is the same as the version of sqlite in your
Conda environment.
