This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 27d62c34d chore(ci): fix wheel builds (#1612)
27d62c34d is described below
commit 27d62c34dc049e91f909b10e53b84cc757b7b1d7
Author: David Li <[email protected]>
AuthorDate: Tue Mar 12 17:03:25 2024 -0400
chore(ci): fix wheel builds (#1612)
- Update versions to match upstream
- Add 3.8 support since upstream uses 3.8 as the base
- Adjust CI to test both 3.8 and 3.12
Fixes #1610.
---------
Co-authored-by: Sutou Kouhei <[email protected]>
---
.env | 4 +-
.github/workflows/packaging.yml | 68 +++++++++++++++++++++-
ci/conda_env_python.txt | 1 +
ci/scripts/install_python.sh | 12 ++--
ci/scripts/python_sdist_test.sh | 2 +-
ci/scripts/python_wheel_unix_test.sh | 2 +-
ci/scripts/python_wheel_windows_test.bat | 2 +-
.../adbc_driver_flightsql/__init__.py | 7 ++-
python/adbc_driver_flightsql/pyproject.toml | 3 +-
.../adbc_driver_manager/_lib.pyx | 2 +-
python/adbc_driver_manager/pyproject.toml | 2 +-
.../adbc_driver_postgresql/__init__.py | 7 ++-
python/adbc_driver_postgresql/pyproject.toml | 3 +-
.../adbc_driver_postgresql/tests/test_lowlevel.py | 4 +-
.../adbc_driver_snowflake/__init__.py | 7 ++-
python/adbc_driver_snowflake/pyproject.toml | 3 +-
.../adbc_driver_sqlite/__init__.py | 7 ++-
python/adbc_driver_sqlite/pyproject.toml | 3 +-
18 files changed, 105 insertions(+), 34 deletions(-)
diff --git a/.env b/.env
index 76c2a65a6..3f56243c1 100644
--- a/.env
+++ b/.env
@@ -31,7 +31,7 @@ ARCH_CONDA_FORGE=linux_64_
JDK=8
MANYLINUX=2014
MAVEN=3.5.4
-PYTHON=3.10
+PYTHON=3.8
GO=1.19.5
ARROW_MAJOR_VERSION=14
DOTNET=7.0
@@ -40,7 +40,7 @@ DOTNET=7.0
# ci/scripts/install_vcpkg.sh script. Keep in sync with apache/arrow .env.
# When updating, also update the docs, which list the version of libpq/SQLite
# that vcpkg (and hence our wheels) ship
-VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1"
+VCPKG="a42af01b72c28a8e1d7b48107b33e4f286a55ef6"
# These are used to tell tests where to find services for integration testing.
# They are valid if the services are started with the docker-compose config.
diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml
index d5241a8f3..4be32878e 100644
--- a/.github/workflows/packaging.yml
+++ b/.github/workflows/packaging.yml
@@ -544,16 +544,46 @@ jobs:
adbc/python/adbc_driver_sqlite/repaired_wheels/*.whl
adbc/python/adbc_driver_snowflake/repaired_wheels/*.whl
- - name: Test wheel
+ - name: Test wheel 3.8
+ env:
+ ARCH: ${{ matrix.arch }}
+ MANYLINUX: ${{ matrix.manylinux_version }}
+ run: |
+ pushd adbc
+ env PYTHON=3.8 docker compose run python-wheel-manylinux-test
+
+ - name: Test wheel 3.9
env:
ARCH: ${{ matrix.arch }}
MANYLINUX: ${{ matrix.manylinux_version }}
run: |
pushd adbc
env PYTHON=3.9 docker compose run python-wheel-manylinux-test
+
+ - name: Test wheel 3.10
+ env:
+ ARCH: ${{ matrix.arch }}
+ MANYLINUX: ${{ matrix.manylinux_version }}
+ run: |
+ pushd adbc
env PYTHON=3.10 docker compose run python-wheel-manylinux-test
+
+ - name: Test wheel 3.11
+ env:
+ ARCH: ${{ matrix.arch }}
+ MANYLINUX: ${{ matrix.manylinux_version }}
+ run: |
+ pushd adbc
env PYTHON=3.11 docker compose run python-wheel-manylinux-test
+ - name: Test wheel 3.12
+ env:
+ ARCH: ${{ matrix.arch }}
+ MANYLINUX: ${{ matrix.manylinux_version }}
+ run: |
+ pushd adbc
+ env PYTHON=3.12 docker compose run python-wheel-manylinux-test
+
python-macos:
name: "Python ${{ matrix.arch }} macOS"
runs-on: macos-latest
@@ -618,9 +648,11 @@ jobs:
- name: Install Python
run: |
pushd adbc
+ sudo ci/scripts/install_python.sh macos 3.8
sudo ci/scripts/install_python.sh macos 3.9
sudo ci/scripts/install_python.sh macos 3.10
sudo ci/scripts/install_python.sh macos 3.11
+ sudo ci/scripts/install_python.sh macos 3.12
popd
- name: Build wheel
@@ -645,7 +677,18 @@ jobs:
adbc/python/adbc_driver_sqlite/repaired_wheels/*.whl
adbc/python/adbc_driver_snowflake/repaired_wheels/*.whl
- - name: Test wheel
+ - name: Test wheel 3.8
+ if: matrix.arch == 'amd64'
+ run: |
+ pushd adbc
+
+ /Library/Frameworks/Python.framework/Versions/3.8/bin/python3.8 -m
venv test-env-38
+ source test-env-38/bin/activate
+ export PYTHON_VERSION=3.8
+ ./ci/scripts/python_wheel_unix_test.sh $(pwd)
+ deactivate
+
+ - name: Test wheel 3.9
if: matrix.arch == 'amd64'
run: |
pushd adbc
@@ -656,19 +699,38 @@ jobs:
./ci/scripts/python_wheel_unix_test.sh $(pwd)
deactivate
+ - name: Test wheel 3.10
+ if: matrix.arch == 'amd64'
+ run: |
+ pushd adbc
+
/Library/Frameworks/Python.framework/Versions/3.10/bin/python3.10 -m
venv test-env-310
source test-env-310/bin/activate
export PYTHON_VERSION=3.10
./ci/scripts/python_wheel_unix_test.sh $(pwd)
deactivate
+ - name: Test wheel 3.11
+ if: matrix.arch == 'amd64'
+ run: |
+ pushd adbc
+
/Library/Frameworks/Python.framework/Versions/3.11/bin/python3.11 -m
venv test-env-311
source test-env-311/bin/activate
export PYTHON_VERSION=3.11
./ci/scripts/python_wheel_unix_test.sh $(pwd)
deactivate
- popd
+ - name: Test wheel 3.12
+ if: matrix.arch == 'amd64'
+ run: |
+ pushd adbc
+
+ /Library/Frameworks/Python.framework/Versions/3.12/bin/python3.12 -m
venv test-env-312
+ source test-env-312/bin/activate
+ export PYTHON_VERSION=3.12
+ ./ci/scripts/python_wheel_unix_test.sh $(pwd)
+ deactivate
python-windows:
name: "Python ${{ matrix.python_version }} Windows"
diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt
index 78fa8557a..cd12d6bff 100644
--- a/ci/conda_env_python.txt
+++ b/ci/conda_env_python.txt
@@ -16,6 +16,7 @@
# under the License.
Cython
+importlib-resources
pandas
pyarrow>=8.0.0
pyright
diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh
index cb0d47d30..8eda08fe7 100755
--- a/ci/scripts/install_python.sh
+++ b/ci/scripts/install_python.sh
@@ -19,7 +19,7 @@
# From apache/arrow/ci/scripts/install_python.sh
-set -eu
+set -euxo pipefail
declare -A platforms
platforms=([windows]=Windows
@@ -27,11 +27,11 @@ platforms=([windows]=Windows
[linux]=Linux)
declare -A versions
-versions=([3.7]=3.7.9
- [3.8]=3.8.10
+versions=([3.8]=3.8.10
[3.9]=3.9.13
- [3.10]=3.10.8
- [3.11]=3.11.0)
+ [3.10]=3.10.11
+ [3.11]=3.11.8
+ [3.12]=3.12.2)
if [ "$#" -ne 2 ]; then
echo "Usage: $0 <platform> <version>"
@@ -48,7 +48,7 @@ full_version=${versions[$2]}
if [ $platform = "MacOSX" ]; then
echo "Downloading Python installer..."
- if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version"
= "3.11" ]; then
+ if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version"
= "3.11" ] || [ "$version" = "3.12" ]; then
fname="python-${full_version}-macos11.pkg"
else
fname="python-${full_version}-macosx10.9.pkg"
diff --git a/ci/scripts/python_sdist_test.sh b/ci/scripts/python_sdist_test.sh
index 2739476e8..46ce43835 100755
--- a/ci/scripts/python_sdist_test.sh
+++ b/ci/scripts/python_sdist_test.sh
@@ -47,7 +47,7 @@ echo "=== Installing sdists ==="
for component in ${COMPONENTS}; do
pip install --no-deps --force-reinstall
${source_dir}/python/${component}/dist/*.tar.gz
done
-pip install pytest pyarrow pandas protobuf
+pip install importlib-resources pytest pyarrow pandas protobuf
echo "=== (${PYTHON_VERSION}) Testing sdists ==="
test_packages
diff --git a/ci/scripts/python_wheel_unix_test.sh
b/ci/scripts/python_wheel_unix_test.sh
index 87943cee1..15eea984d 100755
--- a/ci/scripts/python_wheel_unix_test.sh
+++ b/ci/scripts/python_wheel_unix_test.sh
@@ -49,7 +49,7 @@ for component in ${COMPONENTS}; do
echo "NOTE: assuming wheels are already installed"
fi
done
-pip install pytest pyarrow pandas protobuf
+pip install importlib-resources pytest pyarrow pandas protobuf
echo "=== (${PYTHON_VERSION}) Testing wheels ==="
diff --git a/ci/scripts/python_wheel_windows_test.bat
b/ci/scripts/python_wheel_windows_test.bat
index 8c437b7b0..f31e1c371 100644
--- a/ci/scripts/python_wheel_windows_test.bat
+++ b/ci/scripts/python_wheel_windows_test.bat
@@ -27,7 +27,7 @@ FOR %%c IN (adbc_driver_manager adbc_driver_flightsql
adbc_driver_postgresql adb
)
)
-pip install pytest pyarrow pandas protobuf
+pip install importlib-resources pytest pyarrow pandas protobuf
echo "=== (%PYTHON_VERSION%) Testing wheels ==="
diff --git a/python/adbc_driver_flightsql/adbc_driver_flightsql/__init__.py
b/python/adbc_driver_flightsql/adbc_driver_flightsql/__init__.py
index 7d45adf0b..eeaad2226 100644
--- a/python/adbc_driver_flightsql/adbc_driver_flightsql/__init__.py
+++ b/python/adbc_driver_flightsql/adbc_driver_flightsql/__init__.py
@@ -176,16 +176,17 @@ def connect(
)
[email protected]
[email protected]_cache
def _driver_path() -> str:
- import importlib.resources
import pathlib
import sys
+ import importlib_resources
+
driver = "adbc_driver_flightsql"
# Wheels bundle the shared library
- root = importlib.resources.files(driver)
+ root = importlib_resources.files(driver)
# The filename is always the same regardless of platform
entrypoint = root.joinpath(f"lib{driver}.so")
if entrypoint.is_file():
diff --git a/python/adbc_driver_flightsql/pyproject.toml
b/python/adbc_driver_flightsql/pyproject.toml
index 35362655e..c7de2df10 100644
--- a/python/adbc_driver_flightsql/pyproject.toml
+++ b/python/adbc_driver_flightsql/pyproject.toml
@@ -21,10 +21,11 @@ description = "An ADBC driver for working with Apache Arrow
Flight SQL."
authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
license = {text = "Apache-2.0"}
readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.8"
dynamic = ["version"]
dependencies = [
"adbc-driver-manager",
+ "importlib-resources >=1.3",
]
[project.optional-dependencies]
diff --git a/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx
b/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx
index b96ea6daf..cf85e226c 100644
--- a/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx
+++ b/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx
@@ -1498,7 +1498,7 @@ cdef extern from "_blocking_impl.h" nogil:
c_string
CClearBlockingCallback"pyadbc_driver_manager::ClearBlockingCallback"()
[email protected]
[email protected]_cache
def _init_blocking_call():
error = bytes(CInitBlockingCallback()).decode("utf-8")
if error:
diff --git a/python/adbc_driver_manager/pyproject.toml
b/python/adbc_driver_manager/pyproject.toml
index 16f7d00ee..a8b1b130a 100644
--- a/python/adbc_driver_manager/pyproject.toml
+++ b/python/adbc_driver_manager/pyproject.toml
@@ -21,7 +21,7 @@ description = "A generic entrypoint for ADBC drivers."
authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
license = {text = "Apache-2.0"}
readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.8"
dynamic = ["version"]
dependencies = ["typing-extensions"]
diff --git a/python/adbc_driver_postgresql/adbc_driver_postgresql/__init__.py
b/python/adbc_driver_postgresql/adbc_driver_postgresql/__init__.py
index ac311b641..467794a78 100644
--- a/python/adbc_driver_postgresql/adbc_driver_postgresql/__init__.py
+++ b/python/adbc_driver_postgresql/adbc_driver_postgresql/__init__.py
@@ -40,16 +40,17 @@ def connect(uri: str) -> adbc_driver_manager.AdbcDatabase:
return adbc_driver_manager.AdbcDatabase(driver=_driver_path(), uri=uri)
[email protected]
[email protected]_cache
def _driver_path() -> str:
- import importlib.resources
import pathlib
import sys
+ import importlib_resources
+
driver = "adbc_driver_postgresql"
# Wheels bundle the shared library
- root = importlib.resources.files(driver)
+ root = importlib_resources.files(driver)
# The filename is always the same regardless of platform
entrypoint = root.joinpath(f"lib{driver}.so")
if entrypoint.is_file():
diff --git a/python/adbc_driver_postgresql/pyproject.toml
b/python/adbc_driver_postgresql/pyproject.toml
index 298c8b0cb..8eb48e2ad 100644
--- a/python/adbc_driver_postgresql/pyproject.toml
+++ b/python/adbc_driver_postgresql/pyproject.toml
@@ -21,10 +21,11 @@ description = "A libpq-based ADBC driver for working with
PostgreSQL."
authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
license = {text = "Apache-2.0"}
readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.8"
dynamic = ["version"]
dependencies = [
"adbc-driver-manager",
+ "importlib-resources >=1.3",
]
[project.optional-dependencies]
diff --git a/python/adbc_driver_postgresql/tests/test_lowlevel.py
b/python/adbc_driver_postgresql/tests/test_lowlevel.py
index 8ecaf1a71..98a355bb1 100644
--- a/python/adbc_driver_postgresql/tests/test_lowlevel.py
+++ b/python/adbc_driver_postgresql/tests/test_lowlevel.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-import collections.abc
+import typing
import pyarrow
import pytest
@@ -27,7 +27,7 @@ import adbc_driver_postgresql
@pytest.fixture
def postgres(
postgres_uri: str,
-) -> collections.abc.Generator[adbc_driver_manager.AdbcConnection, None, None]:
+) -> typing.Generator[adbc_driver_manager.AdbcConnection, None, None]:
with adbc_driver_postgresql.connect(postgres_uri) as db:
with adbc_driver_manager.AdbcConnection(db) as conn:
yield conn
diff --git a/python/adbc_driver_snowflake/adbc_driver_snowflake/__init__.py
b/python/adbc_driver_snowflake/adbc_driver_snowflake/__init__.py
index 32dae68c8..1b8e3b56f 100644
--- a/python/adbc_driver_snowflake/adbc_driver_snowflake/__init__.py
+++ b/python/adbc_driver_snowflake/adbc_driver_snowflake/__init__.py
@@ -138,16 +138,17 @@ def connect(
return adbc_driver_manager.AdbcDatabase(driver=_driver_path(), **kwargs)
[email protected]
[email protected]_cache
def _driver_path() -> str:
- import importlib.resources
import pathlib
import sys
+ import importlib_resources
+
driver = "adbc_driver_snowflake"
# Wheels bundle the shared library
- root = importlib.resources.files(driver)
+ root = importlib_resources.files(driver)
# The filename is always the same regardless of platform
entrypoint = root.joinpath(f"lib{driver}.so")
if entrypoint.is_file():
diff --git a/python/adbc_driver_snowflake/pyproject.toml
b/python/adbc_driver_snowflake/pyproject.toml
index 884535fd2..492e1bb02 100644
--- a/python/adbc_driver_snowflake/pyproject.toml
+++ b/python/adbc_driver_snowflake/pyproject.toml
@@ -21,10 +21,11 @@ description = "An ADBC driver for working with Snowflake."
authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
license = {text = "Apache-2.0"}
readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.8"
dynamic = ["version"]
dependencies = [
"adbc-driver-manager",
+ "importlib-resources >=1.3",
]
[project.optional-dependencies]
diff --git a/python/adbc_driver_sqlite/adbc_driver_sqlite/__init__.py
b/python/adbc_driver_sqlite/adbc_driver_sqlite/__init__.py
index 255ecf6d4..b85f1fc83 100644
--- a/python/adbc_driver_sqlite/adbc_driver_sqlite/__init__.py
+++ b/python/adbc_driver_sqlite/adbc_driver_sqlite/__init__.py
@@ -60,16 +60,17 @@ def connect(uri: typing.Optional[str] = None) ->
adbc_driver_manager.AdbcDatabas
return adbc_driver_manager.AdbcDatabase(driver=_driver_path(), uri=uri)
[email protected]
[email protected]_cache
def _driver_path() -> str:
- import importlib.resources
import pathlib
import sys
+ import importlib_resources
+
driver = "adbc_driver_sqlite"
# Wheels bundle the shared library
- root = importlib.resources.files(driver)
+ root = importlib_resources.files(driver)
# The filename is always the same regardless of platform
entrypoint = root.joinpath(f"lib{driver}.so")
if entrypoint.is_file():
diff --git a/python/adbc_driver_sqlite/pyproject.toml
b/python/adbc_driver_sqlite/pyproject.toml
index 282cccf4e..6904b6973 100644
--- a/python/adbc_driver_sqlite/pyproject.toml
+++ b/python/adbc_driver_sqlite/pyproject.toml
@@ -21,10 +21,11 @@ description = "An ADBC driver for working with SQLite."
authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
license = {text = "Apache-2.0"}
readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.8"
dynamic = ["version"]
dependencies = [
"adbc-driver-manager",
+ "importlib-resources >=1.3",
]
[project.optional-dependencies]
