This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 4de1b4f89 feat(python): support free-threading (#3575)
4de1b4f89 is described below
commit 4de1b4f890f7570b0e4e94f32cbf7ebf3fb08421
Author: David Li <[email protected]>
AuthorDate: Tue Oct 14 16:41:53 2025 +0900
feat(python): support free-threading (#3575)
Closes https://github.com/apache/arrow-adbc/issues/2076.
Closes https://github.com/apache/arrow-adbc/issues/3548.
---
.github/workflows/packaging.yml | 9 ++++
ci/scripts/python_wheel_unix_test.sh | 38 +++++++++-----
compose.yaml | 7 +++
.../adbc_driver_manager/_backward.pyx | 1 +
.../adbc_driver_manager/_lib.pyx | 1 +
.../adbc_driver_manager/_reader.pyx | 1 +
python/adbc_driver_manager/pyproject.toml | 2 +-
.../_backward.pyx => tests/conftest.py} | 30 +++--------
python/adbc_driver_manager/tests/test_blocking.py | 10 ++++
python/adbc_driver_manager/tests/test_dbapi.py | 25 ---------
.../tests/test_dbapi_polars_nopyarrow.py | 11 ++--
python/adbc_driver_manager/tests/test_lowlevel.py | 60 +++++++++++-----------
.../_backward.pyx => tests/test_polars.py} | 47 ++++++++---------
13 files changed, 115 insertions(+), 127 deletions(-)
diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml
index 37b576ae6..0cb59b87a 100644
--- a/.github/workflows/packaging.yml
+++ b/.github/workflows/packaging.yml
@@ -735,6 +735,15 @@ jobs:
# pushd adbc
# env PYTHON=3.14 docker compose run python-wheel-manylinux-test
+ # TODO(lidavidm): once we support 3.14, only test 3.14t
+ - name: Test wheel 3.13t
+ env:
+ ARCH: ${{ matrix.arch }}
+ PLATFORM: ${{ matrix.platform }}
+ run: |
+ pushd adbc
+ env PYTHON=3.13t docker compose run
python-wheel-manylinux-freethreaded-test
+
python-macos:
name: "Python ${{ matrix.arch }} macOS"
runs-on: ${{ matrix.os }}
diff --git a/ci/scripts/python_wheel_unix_test.sh
b/ci/scripts/python_wheel_unix_test.sh
index ffc345a0d..e279ef7bf 100755
--- a/ci/scripts/python_wheel_unix_test.sh
+++ b/ci/scripts/python_wheel_unix_test.sh
@@ -32,32 +32,44 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd
)"
source "${script_dir}/python_util.sh"
echo "=== (${PYTHON_VERSION}) Installing wheels ==="
+
+PYTHON_TAG=cp$(python -c "import sysconfig;
print(sysconfig.get_python_version().replace('.', ''))")
+PYTHON_FLAGS=$(python -c "import sysconfig;
print(sysconfig.get_config_var('abiflags'))")
+
for component in ${COMPONENTS}; do
if [[ "${component}" = "adbc_driver_manager" ]]; then
- PYTHON_TAG=cp$(python -c "import sysconfig;
print(sysconfig.get_python_version().replace('.', ''))")
- # We only want cp313-cp313 and not cp313-cp313t (for example)
- PYTHON_TAG="${PYTHON_TAG}-${PYTHON_TAG}"
+ # Don't pick up cp313-cp313t when we wanted cp313-cp313 or vice versa
(for example)
+ WHEEL_TAG="${PYTHON_TAG}-${PYTHON_TAG}${PYTHON_FLAGS}"
else
- PYTHON_TAG=py3-none
+ WHEEL_TAG=py3-none
fi
if [[ -d ${source_dir}/python/${component}/repaired_wheels/ ]]; then
- pip install --no-deps --force-reinstall \
-
${source_dir}/python/${component}/repaired_wheels/*-${PYTHON_TAG}-*.whl
+ python -m pip install --no-deps --force-reinstall \
+
${source_dir}/python/${component}/repaired_wheels/*-${WHEEL_TAG}-*.whl
elif [[ -d ${source_dir}/python/${component}/dist/ ]]; then
- pip install --no-deps --force-reinstall \
- ${source_dir}/python/${component}/dist/*-${PYTHON_TAG}-*.whl
+ python -m pip install --no-deps --force-reinstall \
+ ${source_dir}/python/${component}/dist/*-${WHEEL_TAG}-*.whl
else
echo "NOTE: assuming wheels are already installed"
fi
done
-pip install importlib-resources pytest pyarrow pandas polars protobuf
+python -m pip install importlib-resources pytest pyarrow pandas protobuf
+if [[ -z "${PYTHON_FLAGS}" ]]; then
+ # polars does not support freethreading and will try to build from source
+ python -m pip install polars
+fi
echo "=== (${PYTHON_VERSION}) Testing wheels ==="
test_packages
-echo "=== (${PYTHON_VERSION}) Testing wheels (no PyArrow) ==="
-pip uninstall -y pyarrow
-export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k pyarrowless"
-test_packages_pyarrowless
+if [[ -z "${PYTHON_FLAGS}" ]]; then
+ echo "=== (${PYTHON_VERSION}) Testing wheels (no PyArrow) ==="
+ python -m pip uninstall -y pyarrow
+ export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k pyarrowless"
+ test_packages_pyarrowless
+else
+ echo "Freethreading build, skipping pyarrowless tests"
+ echo "(polars does not yet support freethreading)"
+fi
diff --git a/compose.yaml b/compose.yaml
index b020c4b9a..1f8388b25 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -209,6 +209,13 @@ services:
- .:/adbc:delegated
command: /adbc/ci/scripts/python_wheel_unix_test.sh /adbc
+ python-wheel-manylinux-freethreaded-test:
+ image: ghcr.io/astral-sh/uv:trixie-slim
+ platform: ${PLATFORM}
+ volumes:
+ - .:/adbc:delegated
+ command: bash -c 'uv venv --python ${PYTHON} && source
./.venv/bin/activate && python -m ensurepip && PYTHON_GIL=0
/adbc/ci/scripts/python_wheel_unix_test.sh /adbc'
+
###################### Test database environments
############################
dremio:
diff --git a/python/adbc_driver_manager/adbc_driver_manager/_backward.pyx
b/python/adbc_driver_manager/adbc_driver_manager/_backward.pyx
index 0527d1c4f..d5ac9f8c0 100644
--- a/python/adbc_driver_manager/adbc_driver_manager/_backward.pyx
+++ b/python/adbc_driver_manager/adbc_driver_manager/_backward.pyx
@@ -16,6 +16,7 @@
# under the License.
# cython: language_level = 3
+# cython: freethreading_compatible=True
"""
For debugging, install crash handlers that print a backtrace.
diff --git a/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx
b/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx
index a34613b24..ac98e813f 100644
--- a/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx
+++ b/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx
@@ -16,6 +16,7 @@
# under the License.
# cython: language_level = 3
+# cython: freethreading_compatible=True
"""Low-level ADBC API."""
diff --git a/python/adbc_driver_manager/adbc_driver_manager/_reader.pyx
b/python/adbc_driver_manager/adbc_driver_manager/_reader.pyx
index c3ea39554..43c5f9870 100644
--- a/python/adbc_driver_manager/adbc_driver_manager/_reader.pyx
+++ b/python/adbc_driver_manager/adbc_driver_manager/_reader.pyx
@@ -16,6 +16,7 @@
# under the License.
# cython: language_level = 3
+# cython: freethreading_compatible=True
import pyarrow
from cython.operator cimport dereference as deref
diff --git a/python/adbc_driver_manager/pyproject.toml
b/python/adbc_driver_manager/pyproject.toml
index 704707926..32a9d42b4 100644
--- a/python/adbc_driver_manager/pyproject.toml
+++ b/python/adbc_driver_manager/pyproject.toml
@@ -35,7 +35,7 @@ homepage = "https://arrow.apache.org/adbc/";
repository = "https://github.com/apache/arrow-adbc";
[build-system]
-requires = ["Cython", "setuptools >= 77.0.0"]
+requires = ["Cython >= 3.1.0", "setuptools >= 77.0.0"]
build-backend = "setuptools.build_meta"
[tool.cibuildwheel]
diff --git a/python/adbc_driver_manager/adbc_driver_manager/_backward.pyx
b/python/adbc_driver_manager/tests/conftest.py
similarity index 59%
copy from python/adbc_driver_manager/adbc_driver_manager/_backward.pyx
copy to python/adbc_driver_manager/tests/conftest.py
index 0527d1c4f..e08ad3134 100644
--- a/python/adbc_driver_manager/adbc_driver_manager/_backward.pyx
+++ b/python/adbc_driver_manager/tests/conftest.py
@@ -15,30 +15,14 @@
# specific language governing permissions and limitations
# under the License.
-# cython: language_level = 3
+import typing
-"""
-For debugging, install crash handlers that print a backtrace.
-"""
+import pytest
-import threading
+from adbc_driver_manager import dbapi
-cdef extern from "backward.hpp" nogil:
- cdef struct CSignalHandling"backward::SignalHandling":
- pass
-
-cdef class _SignalHandling:
- cdef CSignalHandling _c_signal_handler
-
-
-_CRASH_HANDLER = None
-_CRASH_HANDLER_LOCK = threading.Lock()
-
-
-def _install_crash_handler():
- global _CRASH_HANDLER
- with _CRASH_HANDLER_LOCK:
- if _CRASH_HANDLER:
- return
- _CRASH_HANDLER = _SignalHandling()
[email protected]
+def sqlite() -> typing.Generator[dbapi.Connection, None, None]:
+ with dbapi.connect(driver="adbc_driver_sqlite") as conn:
+ yield conn
diff --git a/python/adbc_driver_manager/tests/test_blocking.py
b/python/adbc_driver_manager/tests/test_blocking.py
index ea7007736..820104c20 100644
--- a/python/adbc_driver_manager/tests/test_blocking.py
+++ b/python/adbc_driver_manager/tests/test_blocking.py
@@ -24,6 +24,7 @@ having to send the signal, so this tests the handler itself
instead.
import os
import signal
+import sys
import threading
import time
@@ -97,6 +98,15 @@ def test_blocking_raise():
def test_cancel_raise():
event = threading.Event()
+ def _blocking(event):
+ _send_sigint()
+ event.wait()
+ # Under freethreaded python, _blocking ends before _cancel finishes
+ # and raises the exception, so the exception ends up getting thrown
+ # away; sleep a bit to prevent that
+ if hasattr(sys, "_is_gil_enabled") and not getattr(sys,
"_is_gil_enabled")():
+ time.sleep(5)
+
def _cancel():
event.set()
raise ValueError("expected error")
diff --git a/python/adbc_driver_manager/tests/test_dbapi.py
b/python/adbc_driver_manager/tests/test_dbapi.py
index 72699b13e..72ae18f9e 100644
--- a/python/adbc_driver_manager/tests/test_dbapi.py
+++ b/python/adbc_driver_manager/tests/test_dbapi.py
@@ -18,8 +18,6 @@
import pathlib
import pandas
-import polars
-import polars.testing
import pyarrow
import pyarrow.dataset
import pytest
@@ -28,13 +26,6 @@ from pandas.testing import assert_frame_equal
from adbc_driver_manager import dbapi
[email protected]
-def sqlite():
- """Dynamically load the SQLite driver."""
- with dbapi.connect(driver="adbc_driver_sqlite") as conn:
- yield conn
-
-
def test_type_objects():
assert dbapi.NUMBER == pyarrow.int64()
assert pyarrow.int64() == dbapi.NUMBER
@@ -281,22 +272,6 @@ def test_query_fetch_df(sqlite):
)
[email protected]
-def test_query_fetch_polars(sqlite):
- with sqlite.cursor() as cur:
- cur.execute("SELECT 1, 'foo' AS foo, 2.0")
- polars.testing.assert_frame_equal(
- cur.fetch_polars(),
- polars.DataFrame(
- {
- "1": [1],
- "foo": ["foo"],
- "2.0": [2.0],
- }
- ),
- )
-
-
@pytest.mark.sqlite
@pytest.mark.parametrize(
"parameters",
diff --git a/python/adbc_driver_manager/tests/test_dbapi_polars_nopyarrow.py
b/python/adbc_driver_manager/tests/test_dbapi_polars_nopyarrow.py
index da86fd3a1..7bf7476f2 100644
--- a/python/adbc_driver_manager/tests/test_dbapi_polars_nopyarrow.py
+++ b/python/adbc_driver_manager/tests/test_dbapi_polars_nopyarrow.py
@@ -18,12 +18,13 @@
import os
import typing
-import polars
-import polars.testing
import pytest
from adbc_driver_manager import dbapi
+polars = pytest.importorskip("polars")
+polars.testing = pytest.importorskip("polars.testing")
+
pytestmark = pytest.mark.pyarrowless
@@ -39,12 +40,6 @@ def no_pyarrow() -> None:
pytest.skip("Skipping because pyarrow is installed")
[email protected]
-def sqlite() -> typing.Generator[dbapi.Connection, None, None]:
- with dbapi.connect(driver="adbc_driver_sqlite") as conn:
- yield conn
-
-
@pytest.mark.parametrize(
"data",
[
diff --git a/python/adbc_driver_manager/tests/test_lowlevel.py
b/python/adbc_driver_manager/tests/test_lowlevel.py
index e1cb503dc..74a28bca3 100644
--- a/python/adbc_driver_manager/tests/test_lowlevel.py
+++ b/python/adbc_driver_manager/tests/test_lowlevel.py
@@ -24,7 +24,7 @@ import adbc_driver_manager
@pytest.fixture
-def sqlite():
+def sqlite_raw():
"""Dynamically load the SQLite driver."""
with adbc_driver_manager.AdbcDatabase(driver="adbc_driver_sqlite") as db:
with adbc_driver_manager.AdbcConnection(db) as conn:
@@ -99,8 +99,8 @@ def test_error_mapping():
@pytest.mark.sqlite
-def test_database_set_options(sqlite):
- db, _ = sqlite
+def test_database_set_options(sqlite_raw):
+ db, _ = sqlite_raw
with pytest.raises(
adbc_driver_manager.NotSupportedError,
match="Unknown database option foo='bar'",
@@ -115,8 +115,8 @@ def test_database_set_options(sqlite):
@pytest.mark.sqlite
-def test_connection_get_info(sqlite):
- _, conn = sqlite
+def test_connection_get_info(sqlite_raw):
+ _, conn = sqlite_raw
codes = [
adbc_driver_manager.AdbcInfoCode.VENDOR_NAME,
adbc_driver_manager.AdbcInfoCode.VENDOR_VERSION.value,
@@ -139,8 +139,8 @@ def test_connection_get_info(sqlite):
@pytest.mark.sqlite
-def test_connection_get_objects(sqlite):
- _, conn = sqlite
+def test_connection_get_objects(sqlite_raw):
+ _, conn = sqlite_raw
data = pyarrow.record_batch(
[
[1, 2, 3, 4],
@@ -168,8 +168,8 @@ def test_connection_get_objects(sqlite):
@pytest.mark.sqlite
-def test_connection_get_table_schema(sqlite):
- _, conn = sqlite
+def test_connection_get_table_schema(sqlite_raw):
+ _, conn = sqlite_raw
data = pyarrow.record_batch(
[
[1, 2, 3, 4],
@@ -187,23 +187,23 @@ def test_connection_get_table_schema(sqlite):
@pytest.mark.sqlite
-def test_connection_get_table_types(sqlite):
- _, conn = sqlite
+def test_connection_get_table_types(sqlite_raw):
+ _, conn = sqlite_raw
handle = conn.get_table_types()
table = _import(handle).read_all()
assert "table" in table[0].to_pylist()
@pytest.mark.sqlite
-def test_connection_read_partition(sqlite):
- _, conn = sqlite
+def test_connection_read_partition(sqlite_raw):
+ _, conn = sqlite_raw
with pytest.raises(adbc_driver_manager.NotSupportedError):
conn.read_partition(b"")
@pytest.mark.sqlite
-def test_connection_set_options(sqlite):
- _, conn = sqlite
+def test_connection_set_options(sqlite_raw):
+ _, conn = sqlite_raw
with pytest.raises(
adbc_driver_manager.NotSupportedError,
match="Unknown connection option foo='bar'",
@@ -218,8 +218,8 @@ def test_connection_set_options(sqlite):
@pytest.mark.sqlite
-def test_statement_query(sqlite):
- _, conn = sqlite
+def test_statement_query(sqlite_raw):
+ _, conn = sqlite_raw
with adbc_driver_manager.AdbcStatement(conn) as stmt:
stmt.set_sql_query("SELECT 1")
handle, _ = stmt.execute_query()
@@ -228,8 +228,8 @@ def test_statement_query(sqlite):
@pytest.mark.sqlite
-def test_statement_prepared(sqlite):
- _, conn = sqlite
+def test_statement_prepared(sqlite_raw):
+ _, conn = sqlite_raw
with adbc_driver_manager.AdbcStatement(conn) as stmt:
stmt.set_sql_query("SELECT ?")
stmt.prepare()
@@ -241,8 +241,8 @@ def test_statement_prepared(sqlite):
@pytest.mark.sqlite
-def test_statement_ingest(sqlite):
- _, conn = sqlite
+def test_statement_ingest(sqlite_raw):
+ _, conn = sqlite_raw
data = pyarrow.record_batch(
[
[1, 2, 3, 4],
@@ -262,8 +262,8 @@ def test_statement_ingest(sqlite):
@pytest.mark.sqlite
-def test_statement_adbc_prepare(sqlite):
- _, conn = sqlite
+def test_statement_adbc_prepare(sqlite_raw):
+ _, conn = sqlite_raw
with adbc_driver_manager.AdbcStatement(conn) as stmt:
stmt.set_sql_query("SELECT 1")
stmt.prepare()
@@ -282,8 +282,8 @@ def test_statement_adbc_prepare(sqlite):
@pytest.mark.sqlite
-def test_statement_autocommit(sqlite):
- _, conn = sqlite
+def test_statement_autocommit(sqlite_raw):
+ _, conn = sqlite_raw
# Autocommit enabled by default
with pytest.raises(adbc_driver_manager.ProgrammingError) as errholder:
@@ -356,8 +356,8 @@ def test_statement_autocommit(sqlite):
@pytest.mark.sqlite
-def test_statement_set_options(sqlite):
- _, conn = sqlite
+def test_statement_set_options(sqlite_raw):
+ _, conn = sqlite_raw
with adbc_driver_manager.AdbcStatement(conn) as stmt:
with pytest.raises(
@@ -374,7 +374,7 @@ def test_statement_set_options(sqlite):
@pytest.mark.sqlite
-def test_child_tracking(sqlite):
+def test_child_tracking(sqlite_raw):
with adbc_driver_manager.AdbcDatabase(driver="adbc_driver_sqlite") as db:
with adbc_driver_manager.AdbcConnection(db) as conn:
with adbc_driver_manager.AdbcStatement(conn):
@@ -395,8 +395,8 @@ def test_child_tracking(sqlite):
@pytest.mark.sqlite
-def test_pycapsule(sqlite):
- _, conn = sqlite
+def test_pycapsule(sqlite_raw):
+ _, conn = sqlite_raw
handle = conn.get_table_types()
with pyarrow.RecordBatchReader._import_from_c_capsule(
handle.__arrow_c_stream__()
diff --git a/python/adbc_driver_manager/adbc_driver_manager/_backward.pyx
b/python/adbc_driver_manager/tests/test_polars.py
similarity index 59%
copy from python/adbc_driver_manager/adbc_driver_manager/_backward.pyx
copy to python/adbc_driver_manager/tests/test_polars.py
index 0527d1c4f..a48740fa8 100644
--- a/python/adbc_driver_manager/adbc_driver_manager/_backward.pyx
+++ b/python/adbc_driver_manager/tests/test_polars.py
@@ -15,30 +15,23 @@
# specific language governing permissions and limitations
# under the License.
-# cython: language_level = 3
-
-"""
-For debugging, install crash handlers that print a backtrace.
-"""
-
-import threading
-
-cdef extern from "backward.hpp" nogil:
- cdef struct CSignalHandling"backward::SignalHandling":
- pass
-
-
-cdef class _SignalHandling:
- cdef CSignalHandling _c_signal_handler
-
-
-_CRASH_HANDLER = None
-_CRASH_HANDLER_LOCK = threading.Lock()
-
-
-def _install_crash_handler():
- global _CRASH_HANDLER
- with _CRASH_HANDLER_LOCK:
- if _CRASH_HANDLER:
- return
- _CRASH_HANDLER = _SignalHandling()
+import pytest
+
+polars = pytest.importorskip("polars")
+polars.testing = pytest.importorskip("polars.testing")
+
+
[email protected]
+def test_query_fetch_polars(sqlite):
+ with sqlite.cursor() as cur:
+ cur.execute("SELECT 1, 'foo' AS foo, 2.0")
+ polars.testing.assert_frame_equal(
+ cur.fetch_polars(),
+ polars.DataFrame(
+ {
+ "1": [1],
+ "foo": ["foo"],
+ "2.0": [2.0],
+ }
+ ),
+ )
