This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new 61c6917d chore(python): Update Python packaging (#625)
61c6917d is described below
commit 61c6917d27e16b7ffdcff711ce8d7e6ddebf6a5d
Author: Dewey Dunnington <[email protected]>
AuthorDate: Fri Sep 20 16:21:23 2024 -0500
chore(python): Update Python packaging (#625)
This PR updates the Python packaging to build a single nanoarrow library
that is shared amongst the various extension modules and updates the
version of cibuildwheel such that we can release Python 3.13 wheels in
our upcoming release. I'm not sure where the mechanism used to build a
"shared" library is documented, nor am I sure that it is truly a
"shared" library (may be static); however, it seems conceptually better
than the existing system of compiling the source files more than once
for each module that needed them.
---
.github/workflows/python-wheels.yaml | 2 +-
python/setup.py | 75 ++++++++++++++++++------------------
2 files changed, 39 insertions(+), 38 deletions(-)
diff --git a/.github/workflows/python-wheels.yaml
b/.github/workflows/python-wheels.yaml
index acb9e0a6..5c3ff6a6 100644
--- a/.github/workflows/python-wheels.yaml
+++ b/.github/workflows/python-wheels.yaml
@@ -111,7 +111,7 @@ jobs:
python-version: "3.12"
- name: Install cibuildwheel
- run: python -m pip install cibuildwheel==2.19.1
+ run: python -m pip install cibuildwheel==2.21.1
- name: Set nanoarrow Python dev version
if: github.ref == 'refs/heads/main'
diff --git a/python/setup.py b/python/setup.py
index 1b776247..a2335ecc 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -94,55 +94,56 @@ if cuda_toolkit_root:
device_library_dirs.append(str(lib_dirs[0].parent))
-def nanoarrow_extension(
- name, *, nanoarrow_c=False, nanoarrow_device=False, nanoarrow_ipc=False
-):
- sources = ["src/" + name.replace(".", "/") + ".pyx"]
- libraries = []
- library_dirs = []
- include_dirs = ["src/nanoarrow", "vendor"]
- define_macros = list(extra_define_macros)
-
- if nanoarrow_c:
- sources.append("vendor/nanoarrow.c")
-
- if nanoarrow_device:
- sources.append("vendor/nanoarrow_device.c")
- include_dirs.extend(device_include_dirs)
- libraries.extend(device_libraries)
- library_dirs.extend(device_library_dirs)
- define_macros.extend(device_define_macros)
-
- if nanoarrow_ipc:
- sources.extend(["vendor/nanoarrow_ipc.c", "vendor/flatcc.c"])
+# This mechanism to build a static c library against which extensions
+# can be linked is not well documented but is a better solution than
+# simply including these files as sources to the extensions that need
+# them. A more robust solution would be to use Meson or CMake to build
+# the Python extensions since they can both build a shared nanoarrow
+# and link it. This mechanism is the build_clib command available in
+# setuptools (and previously from distutils).
+common_libraries = [
+ [
+ "nanoarrow_python_shared",
+ {
+ "sources": [
+ "vendor/nanoarrow.c",
+ "vendor/nanoarrow_device.c",
+ "vendor/nanoarrow_ipc.c",
+ "vendor/flatcc.c",
+ ],
+ "include_dirs": ["vendor"] + device_include_dirs,
+ "libraries": device_libraries,
+ "library_dirs": device_library_dirs,
+ "macros": extra_define_macros + device_define_macros,
+ },
+ ]
+]
+
+def nanoarrow_extension(name, *, link_device=False):
return Extension(
name=name,
- include_dirs=include_dirs,
+ include_dirs=["vendor", "src/nanoarrow"],
language="c",
- sources=sources,
+ sources=["src/" + name.replace(".", "/") + ".pyx"],
extra_compile_args=extra_compile_args,
extra_link_args=extra_link_args,
- define_macros=define_macros,
- library_dirs=library_dirs,
- libraries=libraries,
+ define_macros=extra_define_macros + device_define_macros,
+ libraries=["nanoarrow_python_shared"] + device_libraries if
link_device else [],
)
setup(
ext_modules=[
nanoarrow_extension("nanoarrow._types"),
- nanoarrow_extension("nanoarrow._utils", nanoarrow_c=True),
- nanoarrow_extension(
- "nanoarrow._device", nanoarrow_c=True, nanoarrow_device=True
- ),
- nanoarrow_extension(
- "nanoarrow._array", nanoarrow_c=True, nanoarrow_device=True
- ),
- nanoarrow_extension("nanoarrow._array_stream", nanoarrow_c=True),
- nanoarrow_extension("nanoarrow._buffer", nanoarrow_c=True),
- nanoarrow_extension("nanoarrow._ipc_lib", nanoarrow_c=True,
nanoarrow_ipc=True),
- nanoarrow_extension("nanoarrow._schema", nanoarrow_c=True),
+ nanoarrow_extension("nanoarrow._utils"),
+ nanoarrow_extension("nanoarrow._device", link_device=True),
+ nanoarrow_extension("nanoarrow._array", link_device=True),
+ nanoarrow_extension("nanoarrow._array_stream"),
+ nanoarrow_extension("nanoarrow._buffer"),
+ nanoarrow_extension("nanoarrow._ipc_lib"),
+ nanoarrow_extension("nanoarrow._schema"),
],
version=version,
+ libraries=common_libraries,
)
