This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new e88f98e ci(c): properly tag wheels with platform (#206)
e88f98e is described below
commit e88f98e47bae6be58d048602875451a58d3c91cf
Author: David Li <[email protected]>
AuthorDate: Thu Dec 1 14:39:22 2022 -0500
ci(c): properly tag wheels with platform (#206)
auditwheel properly tags our frankenwheels with the platform, but delocate
and delvewheel do not. Do this manually to ensure consistency.
---
.github/workflows/packaging-wheels.yml | 14 +++--
c/driver/sqlite/CMakeLists.txt | 10 ++--
ci/scripts/python_util.sh | 12 ++--
ci/scripts/python_wheel_fix_tag.py | 94 +++++++++++++++++++++++++++++++
ci/scripts/python_wheel_unix_build.sh | 17 +++---
ci/scripts/python_wheel_windows_build.bat | 12 +++-
6 files changed, 132 insertions(+), 27 deletions(-)
diff --git a/.github/workflows/packaging-wheels.yml
b/.github/workflows/packaging-wheels.yml
index 19f88e5..b149c2e 100644
--- a/.github/workflows/packaging-wheels.yml
+++ b/.github/workflows/packaging-wheels.yml
@@ -134,6 +134,7 @@ jobs:
path: |
python/adbc_driver_manager/repaired_wheels/*.whl
python/adbc_driver_postgres/repaired_wheels/*.whl
+ python/adbc_driver_sqlite/repaired_wheels/*.whl
- name: Test wheel
shell: bash
@@ -146,7 +147,7 @@ jobs:
shell: bash
if: github.ref == 'refs/heads/main' && (github.event.schedule ||
github.event.inputs.upload_artifacts == true)
run: |
- ./ci/scripts/python_wheel_upload.sh
python/adbc_driver_{manager,postgres}/repaired_wheels/*.whl
+ ./ci/scripts/python_wheel_upload.sh
python/adbc_driver_{manager,postgres,sqlite}/repaired_wheels/*.whl
env:
GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
@@ -210,8 +211,9 @@ jobs:
name: python${{ matrix.python_version }}-macOS
retention-days: 7
path: |
- python/adbc_driver_manager/dist/*.whl
- python/adbc_driver_postgres/dist/*.whl
+ python/adbc_driver_manager/repaired_wheels/*.whl
+ python/adbc_driver_postgres/repaired_wheels/*.whl
+ python/adbc_driver_sqlite/repaired_wheels/*.whl
- name: Test wheel
shell: bash
@@ -224,7 +226,7 @@ jobs:
shell: bash
if: github.ref == 'refs/heads/main' && (github.event.schedule ||
github.event.inputs.upload_artifacts == true)
run: |
- ./ci/scripts/python_wheel_upload.sh
python/adbc_driver_{manager,postgres}/dist/*.whl
+ ./ci/scripts/python_wheel_upload.sh
python/adbc_driver_{manager,postgres,sqlite}/dist/*.whl
env:
GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
@@ -300,6 +302,7 @@ jobs:
path: |
python/adbc_driver_manager/dist/*.whl
python/adbc_driver_postgres/dist/*.whl
+ python/adbc_driver_sqlite/dist/*.whl
- name: Test wheel
shell: cmd
@@ -348,6 +351,7 @@ jobs:
path: |
python/adbc_driver_manager/dist/*.tar.gz
python/adbc_driver_postgres/dist/*.tar.gz
+ python/adbc_driver_sqlite/dist/*.tar.gz
- name: Test sdist
shell: bash
@@ -358,6 +362,6 @@ jobs:
shell: bash
if: github.ref == 'refs/heads/main' && (github.event.schedule ||
github.event.inputs.upload_artifacts == true ||
github.event.inputs.upload_artifacts == 'true')
run: |
- ./ci/scripts/python_wheel_upload.sh
python/adbc_driver_{manager,postgres}/dist/*.tar.gz
+ ./ci/scripts/python_wheel_upload.sh
python/adbc_driver_{manager,postgres,sqlite}/dist/*.tar.gz
env:
GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
diff --git a/c/driver/sqlite/CMakeLists.txt b/c/driver/sqlite/CMakeLists.txt
index a80fd0d..490594c 100644
--- a/c/driver/sqlite/CMakeLists.txt
+++ b/c/driver/sqlite/CMakeLists.txt
@@ -26,14 +26,14 @@ project(adbc_driver_sqlite
LANGUAGES CXX)
include(CTest)
-if(WIN32)
- # XXX: for now, assume vcpkg
+find_package(SQLite3)
+if(SQLite3_FOUND)
+ set(SQLite3_LINK_LIBRARIES SQLite::SQLite3)
+else()
+ # vcpkg
find_package(unofficial-sqlite3 CONFIG REQUIRED)
set(SQLite3_LINK_LIBRARIES unofficial::sqlite3::sqlite3)
set(SQLite3_INCLUDE_DIRS)
-else()
- find_package(SQLite3 REQUIRED)
- set(SQLite3_LINK_LIBRARIES SQLite::SQLite3)
endif()
add_arrow_lib(adbc_driver_sqlite
diff --git a/ci/scripts/python_util.sh b/ci/scripts/python_util.sh
index bd2a01b..c8512a5 100644
--- a/ci/scripts/python_util.sh
+++ b/ci/scripts/python_util.sh
@@ -36,20 +36,16 @@ function build_drivers {
export
ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.so
export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.so
export VCPKG_DEFAULT_TRIPLET="x64-linux-static-release"
-
- # XXX: Patch the portfile
- sed -i
"s|include/postgresql/server/pg_config.h|include/server/pg_config.h|" \
- "${VCPKG_ROOT}/ports/libpq/portfile.cmake"
else # macOS
export
ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.dylib
export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.dylib
export VCPKG_DEFAULT_TRIPLET="x64-osx-static-release"
-
- # XXX: Patch the portfile
- sed -i '.bak'
"s|include/postgresql/server/pg_config.h|include/server/pg_config.h|" \
- "${VCPKG_ROOT}/ports/libpq/portfile.cmake"
fi
+ "${VCPKG_ROOT}/vcpkg" install libpq sqlite3 \
+ --overlay-triplets "${VCPKG_OVERLAY_TRIPLETS}" \
+ --triplet "${VCPKG_DEFAULT_TRIPLET}"
+
echo "=== Building driver/postgres ==="
mkdir -p ${build_dir}/driver/postgres
pushd ${build_dir}/driver/postgres
diff --git a/ci/scripts/python_wheel_fix_tag.py
b/ci/scripts/python_wheel_fix_tag.py
new file mode 100755
index 0000000..890c791
--- /dev/null
+++ b/ci/scripts/python_wheel_fix_tag.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Fix the platform tag in macOS/Windows wheels because
+# delocate/delvewheel don't do the right thing.
+
+import argparse
+import subprocess
+import sys
+import sysconfig
+import tempfile
+from pathlib import Path
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("wheel")
+ args = parser.parse_args()
+
+ plat_tag = sysconfig.get_platform()
+ plat_tag = plat_tag.replace("-", "_")
+ plat_tag = plat_tag.replace(".", "_")
+ print("Using platform tag: ", plat_tag)
+
+ with tempfile.TemporaryDirectory() as dest:
+ dest = Path(dest)
+ subprocess.check_call(
+ [
+ sys.executable,
+ "-m",
+ "wheel",
+ "unpack",
+ "--dest",
+ str(dest),
+ args.wheel,
+ ]
+ )
+
+ (root,) = list(dest.glob("*"))
+
+ metadata_files = list(root.rglob("*.dist-info/WHEEL"))
+ if len(metadata_files) != 1:
+ raise ValueError(f"Expected one WHEEL file, found:
{metadata_files}")
+
+ with metadata_files[0].open() as source:
+ contents = source.read()
+
+ new_contents = contents.replace(
+ "Root-Is-Purelib: true", "Root-Is-Purelib: false"
+ )
+ new_contents = new_contents.replace(
+ "Tag: py3-none-any", f"Tag: py3-none-{plat_tag}"
+ )
+
+ if new_contents == contents:
+ print("No changes made (wheel already properly tagged?)")
+ return
+
+ with metadata_files[0].open("w") as sink:
+ sink.write(new_contents)
+
+ wheel = Path(args.wheel).resolve()
+ subprocess.check_call(
+ [
+ sys.executable,
+ "-m",
+ "wheel",
+ "pack",
+ "--dest-dir",
+ str(wheel.parent),
+ str(root),
+ ]
+ )
+ wheel.unlink()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/ci/scripts/python_wheel_unix_build.sh
b/ci/scripts/python_wheel_unix_build.sh
index 616462c..ad5566d 100755
--- a/ci/scripts/python_wheel_unix_build.sh
+++ b/ci/scripts/python_wheel_unix_build.sh
@@ -53,12 +53,8 @@ function check_wheels {
echo "=== (${PYTHON_VERSION}) Tag $component wheel with
manylinux${MANYLINUX_VERSION} ==="
auditwheel repair "$@" -L . -w repaired_wheels
else # macOS
- echo "=== (${PYTHON_VERSION}) Check $component wheel for unbundled
dependencies ==="
- local -r deps=$(delocate-listdeps dist/$component-*.whl)
- if ! echo $deps | grep -v "python/"; then
- echo "There are unbundled dependencies."
- exit 1
- fi
+ echo "=== (${PYTHON_VERSION}) Tag $component wheel with macOS ==="
+ delocate-wheel -v -k -w repaired_wheels "$@"
fi
}
@@ -72,7 +68,10 @@ check_visibility $ADBC_SQLITE_LIBRARY
# https://github.com/pypa/pip/issues/7555
# Get the latest pip so we have in-tree-build by default
-pip install --upgrade pip auditwheel
+# Get wheel so we can manually invoke bdist_wheel
+python -m pip install --upgrade pip auditwheel delocate setuptools wheel
+
+PLAT_NAME=$(python -c "import sysconfig; print(sysconfig.get_platform())")
for component in $COMPONENTS; do
pushd ${source_dir}/python/$component
@@ -82,9 +81,11 @@ for component in $COMPONENTS; do
echo "=== (${PYTHON_VERSION}) Building $component wheel ==="
# python -m build copies to a tempdir, so we can't reference other files
in the repo
- # https://github.com/pypa/pip/issues/5519
python -m pip wheel -w dist -vvv .
+ # Retag the wheel
+ python "${script_dir}/python_wheel_fix_tag.py" dist/$component-*.whl
+
check_wheels dist/$component-*.whl
popd
diff --git a/ci/scripts/python_wheel_windows_build.bat
b/ci/scripts/python_wheel_windows_build.bat
index 3dcea73..76a598b 100644
--- a/ci/scripts/python_wheel_windows_build.bat
+++ b/ci/scripts/python_wheel_windows_build.bat
@@ -72,7 +72,9 @@ set ADBC_SQLITE_LIBRARY=%build_dir%\bin\adbc_driver_sqlite.dll
popd
-python -m pip install --upgrade pip delvewheel
+python -m pip install --upgrade pip delvewheel wheel
+
+FOR /F %%i IN ('python -c "import sysconfig;
print(sysconfig.get_platform())"') DO set PLAT_NAME=%%i
FOR %%c IN (adbc_driver_manager adbc_driver_postgres adbc_driver_sqlite) DO (
pushd %source_dir%\python\%%c
@@ -80,6 +82,14 @@ FOR %%c IN (adbc_driver_manager adbc_driver_postgres
adbc_driver_sqlite) DO (
echo "=== (%PYTHON_VERSION%) Building %%c wheel ==="
python -m pip wheel -w dist -vvv . || exit /B 1
+ echo "=== (%PYTHON_VERSION%) Re-tag %%c wheel ==="
+ FOR %%w IN (dist\*.whl) DO (
+ python %source_dir%\ci\scripts\python_wheel_fix_tag.py %%w || exit /B 1
+ )
+
+ echo "=== (%PYTHON_VERSION%) Repair %%c wheel ==="
+ python -m pip wheel -w dist -vvv . || exit /B 1
+
FOR %%w IN (dist\*.whl) DO (
delvewheel repair -w dist\ %%w
)
