This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new b636a8f8 ci(python): Add cibuildwheel setup for Python wheels (#353)
b636a8f8 is described below
commit b636a8f88d6a529ff31da6847ac44b584847e5f6
Author: Dewey Dunnington <[email protected]>
AuthorDate: Wed Jan 10 19:52:51 2024 +0000
ci(python): Add cibuildwheel setup for Python wheels (#353)
This PR adds a weekly/workflow dispatch job for building and testing
Python wheels. This required a few housekeeping items:
- Versioning the python package. I used the approach from ADBC, which is
a modified 'miniver'. Basically, just set the version as a string using
a regex replace when needed.
- The bootstrap.py logic was updated to use a proper temporary directory
- Tests were updated to skip instead of fail when pyarrow/numpy are not
available (because I can never remember which platforms they will or
won't install on and the default cibuildwheel grid is large).
- I hadn't tested install from sdist, so a few files were missing from
the manifest.
At least one test doesn't pass on 32-bit Windows (already fixed in
#340). For now I just enabled the version tests to make sure everything
built/linked properly.
---------
Co-authored-by: Joris Van den Bossche <[email protected]>
---
.github/workflows/python-wheels.yaml | 111 +++++++++++++++++++++
dev/release/utils-prepare.sh | 8 ++
python/.gitattributes | 1 +
python/MANIFEST.in | 3 +
python/bootstrap.py | 77 +++++++-------
python/pyproject.toml | 4 +-
python/setup.py | 23 ++++-
python/src/nanoarrow/__init__.py | 1 +
.../nanoarrow/{__init__.py => _static_version.py} | 10 +-
python/src/nanoarrow/_version.py | 50 ++++++++++
python/tests/test_capsules.py | 4 +-
python/tests/test_device.py | 4 +-
python/tests/test_nanoarrow.py | 9 +-
python/{MANIFEST.in => tests/test_version.py} | 17 +++-
14 files changed, 270 insertions(+), 52 deletions(-)
diff --git a/.github/workflows/python-wheels.yaml
b/.github/workflows/python-wheels.yaml
new file mode 100644
index 00000000..4168b96f
--- /dev/null
+++ b/.github/workflows/python-wheels.yaml
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Build Python Wheels
+
+# Build wheels weekly, on commit to main, or when requested
+on:
+ pull_request:
+ branches:
+ - main
+ paths:
+ - '.github/workflows/python-wheels.yaml'
+ - 'python/setup.py'
+ - 'python/pyproject.toml'
+ - 'python/bootstrap.py'
+ - 'python/MANIFEST.in'
+ push:
+ branches:
+ - main
+ workflow_dispatch:
+ schedule:
+ - cron: '6 0 * * 0'
+
+jobs:
+ build_sdist:
+ runs-on: "ubuntu-20.04"
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ - name: Check that cmake is installed
+ run: |
+ cmake --version
+
+ - name: Install packaging tools
+ run: |
+ pip install build twine
+
+ - name: Build sdist
+ run: |
+ cd python
+ python -m build --sdist
+
+ - name: Check install from sdist
+ run: |
+ pip install python/dist/nanoarrow-*.tar.gz
+
+ - name: Test import
+ run: |
+ python -c "import nanoarrow; print(nanoarrow.__version__)"
+
+ - name: Run twine check
+ run: |
+ twine check --strict python/dist/*
+
+ - uses: actions/upload-artifact@v4
+ with:
+ name: sdist
+ path: ./python/dist/nanoarrow-*.tar.gz
+
+ build_wheels:
+ needs: ["build_sdist"]
+ name: Build wheels on ${{ matrix.config.label }}
+ runs-on: ${{ matrix.config.os }}
+ strategy:
+ matrix:
+ config:
+ - {os: "ubuntu-20.04", label: "linux"}
+ - {os: "windows-2019", label: "windows"}
+ - {os: "macOS-11", label: "macOS"}
+ - {os: ["self-hosted", "arm"], label: "linux-arm64"}
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-python@v5
+ if: matrix.config.label != 'linux-arm64'
+ with:
+ python-version: "3.11"
+
+ - name: Install cibuildwheel
+ run: python -m pip install cibuildwheel==2.15.0
+
+ - name: Build wheels
+ run: |
+ python -m cibuildwheel --output-dir wheelhouse python
+ env:
+ CIBW_ARCHS_MACOS: x86_64 arm64
+ # Optional (test suite will pass if these are not available)
+ # Commenting this for now because not all the tests pass yet (fixes
in another PR)
+ # CIBW_BEFORE_TEST: pip install --only-binary ":all:" pyarrow numpy
|| pip install --only-binary ":all:" numpy || true
+ CIBW_TEST_REQUIRES: pytest
+ CIBW_TEST_COMMAND: pytest {package}/tests
+
+ - uses: actions/upload-artifact@v4
+ with:
+ name: wheels-${{ matrix.config.label }}
+ path: ./wheelhouse/*.whl
diff --git a/dev/release/utils-prepare.sh b/dev/release/utils-prepare.sh
index dafab506..97b5d869 100755
--- a/dev/release/utils-prepare.sh
+++ b/dev/release/utils-prepare.sh
@@ -26,11 +26,13 @@ update_versions() {
release)
local version=${base_version}
local docs_version=${base_version}
+ local python_version=${base_version}
local r_version=${base_version}
;;
snapshot)
local version=${next_version}-SNAPSHOT
local docs_version="${next_version} (dev)"
+ local python_version="${next_version}dev"
local r_version="${base_version}.9000"
;;
esac
@@ -46,6 +48,12 @@ update_versions() {
Rscript -e "desc::desc_set(Version = '${r_version}')"
git add DESCRIPTION
popd
+
+ pushd "${NANOARROW_DIR}/python/src/nanoarrow"
+ sed -i.bak -E "s/version = \".+\"/version = \"${python_version}\"/"
_static_version.py
+ rm _static_version.py.bak
+ git add _static_version.py
+ popd
}
set_resolved_issues() {
diff --git a/python/.gitattributes b/python/.gitattributes
new file mode 100644
index 00000000..fea5bc24
--- /dev/null
+++ b/python/.gitattributes
@@ -0,0 +1 @@
+src/nanoarrow/_static_version.py export-subst
diff --git a/python/MANIFEST.in b/python/MANIFEST.in
index 9fc29372..21fdc935 100644
--- a/python/MANIFEST.in
+++ b/python/MANIFEST.in
@@ -19,3 +19,6 @@ exclude bootstrap.py
include src/nanoarrow/nanoarrow.c
include src/nanoarrow/nanoarrow.h
include src/nanoarrow/nanoarrow_c.pxd
+include src/nanoarrow/nanoarrow_device.c
+include src/nanoarrow/nanoarrow_device.h
+include src/nanoarrow/nanoarrow_device_c.pxd
diff --git a/python/bootstrap.py b/python/bootstrap.py
index bbb5d669..b540058a 100644
--- a/python/bootstrap.py
+++ b/python/bootstrap.py
@@ -18,6 +18,9 @@
import os
import re
import shutil
+import subprocess
+import tempfile
+import warnings
# Generate the nanoarrow_c.pxd file used by the Cython extension
@@ -158,7 +161,6 @@ class NanoarrowPxdGenerator:
# any changes from nanoarrow C library sources in the checkout but is not
# strictly necessary for things like installing from GitHub.
def copy_or_generate_nanoarrow_c():
- this_wd = os.getcwd()
this_dir = os.path.abspath(os.path.dirname(__file__))
source_dir = os.path.dirname(this_dir)
@@ -185,42 +187,49 @@ def copy_or_generate_nanoarrow_c():
os.path.join(source_dir, "src", "nanoarrow")
)
has_cmake = os.system("cmake --version") == 0
- build_dir = os.path.join(this_dir, "_cmake")
- if is_in_nanoarrow_repo:
- device_ext_src = os.path.join(
- source_dir, "extensions/nanoarrow_device/src/nanoarrow"
- )
- shutil.copyfile(
- os.path.join(device_ext_src, "nanoarrow_device.h"),
maybe_nanoarrow_device_h
- )
- shutil.copyfile(
- os.path.join(device_ext_src, "nanoarrow_device.c"),
maybe_nanoarrow_device_c
- )
+ with tempfile.TemporaryDirectory() as build_dir:
+ if is_in_nanoarrow_repo:
+ device_ext_src = os.path.join(
+ source_dir, "extensions/nanoarrow_device/src/nanoarrow"
+ )
+ shutil.copyfile(
+ os.path.join(device_ext_src, "nanoarrow_device.h"),
+ maybe_nanoarrow_device_h,
+ )
+ shutil.copyfile(
+ os.path.join(device_ext_src, "nanoarrow_device.c"),
+ maybe_nanoarrow_device_c,
+ )
- if has_cmake and is_cmake_dir and is_in_nanoarrow_repo:
- try:
- os.mkdir(build_dir)
- os.chdir(build_dir)
- os.system(
- "cmake ../.. -DNANOARROW_BUNDLE=ON
-DNANOARROW_NAMESPACE=PythonPkg"
+ if has_cmake and is_cmake_dir and is_in_nanoarrow_repo:
+ try:
+ subprocess.run(
+ [
+ "cmake",
+ "-B",
+ build_dir,
+ "-S",
+ source_dir,
+ "-DNANOARROW_BUNDLE=ON",
+ "-DNANOARROW_NAMESPACE=PythonPkg",
+ ]
+ )
+ subprocess.run(
+ [
+ "cmake",
+ "--install",
+ build_dir,
+ "--prefix",
+ os.path.join(this_dir, "src", "nanoarrow"),
+ ]
+ )
+ except Exception as e:
+ warnings.warn(f"cmake call failed: {e}")
+ else:
+ raise ValueError(
+ "Attempt to build source distribution outside the nanoarrow
repo"
)
- os.system("cmake --install . --prefix=../src/nanoarrow")
- finally:
- if os.path.exists(build_dir):
- # Can fail on Windows with permission issues
- try:
- shutil.rmtree(build_dir)
- except Exception as e:
- print(f"Failed to remove _cmake temp directory: {str(e)}")
- os.chdir(this_wd)
-
- elif is_in_nanoarrow_repo:
- shutil.copyfile()
- else:
- raise ValueError(
- "Attempt to build source distribution outside the nanoarrow repo"
- )
if not os.path.exists(os.path.join(this_dir, "src/nanoarrow/nanoarrow.h")):
raise ValueError("Attempt to vendor nanoarrow.c/h failed")
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 743cebe0..7850a09f 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -18,7 +18,8 @@
[project]
name = "nanoarrow"
-version = "1.0.0-alpha0"
+dynamic = ["version"]
+readme = "README.md"
description = "Python bindings to the nanoarrow C library"
authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
license = {text = "Apache-2.0"}
@@ -34,7 +35,6 @@ repository = "https://github.com/apache/arrow-nanoarrow";
[build-system]
requires = [
"setuptools >= 61.0.0",
- "setuptools-scm",
"Cython"
]
build-backend = "setuptools.build_meta"
diff --git a/python/setup.py b/python/setup.py
index f99d9994..92f8abc9 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -23,6 +23,25 @@ import sys
from setuptools import Extension, setup
+
+# https://github.com/jbweston/miniver
+def get_version(pkg_path):
+ """
+ Load version.py module without importing the whole package.
+
+ Template code from miniver.
+ """
+ from importlib.util import module_from_spec, spec_from_file_location
+
+ spec = spec_from_file_location("version", os.path.join(pkg_path,
"_version.py"))
+ module = module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module.__version__
+
+
+version = get_version("src/nanoarrow")
+
+
# Run bootstrap.py to run cmake generating a fresh bundle based on this
# checkout or copy from ../dist if the caller doesn't have cmake available.
# Note that bootstrap.py won't exist if building from sdist.
@@ -46,6 +65,7 @@ else:
extra_link_args = []
extra_define_macros = []
+
setup(
ext_modules=[
Extension(
@@ -61,5 +81,6 @@ setup(
extra_link_args=extra_link_args,
define_macros=extra_define_macros,
)
- ]
+ ],
+ version=version,
)
diff --git a/python/src/nanoarrow/__init__.py b/python/src/nanoarrow/__init__.py
index 46204d3f..789f4597 100644
--- a/python/src/nanoarrow/__init__.py
+++ b/python/src/nanoarrow/__init__.py
@@ -17,3 +17,4 @@
from ._lib import Array, ArrayStream, ArrayView, Schema, c_version # noqa:
F401
from .lib import array, array_stream, schema, array_view # noqa: F401
+from ._version import __version__ # noqa: F401
diff --git a/python/src/nanoarrow/__init__.py
b/python/src/nanoarrow/_static_version.py
similarity index 73%
copy from python/src/nanoarrow/__init__.py
copy to python/src/nanoarrow/_static_version.py
index 46204d3f..53569eff 100644
--- a/python/src/nanoarrow/__init__.py
+++ b/python/src/nanoarrow/_static_version.py
@@ -14,6 +14,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# This file is part of 'miniver': https://github.com/jbweston/miniver
+
+# Replaced by version-bumping scripts at release time
+version = "0.4.0dev0"
-from ._lib import Array, ArrayStream, ArrayView, Schema, c_version # noqa:
F401
-from .lib import array, array_stream, schema, array_view # noqa: F401
+# These values are only set if the distribution was created with 'git archive'
+refnames = "$Format:%D$"
+git_hash = "$Format:%h$"
diff --git a/python/src/nanoarrow/_version.py b/python/src/nanoarrow/_version.py
new file mode 100644
index 00000000..750dbb5b
--- /dev/null
+++ b/python/src/nanoarrow/_version.py
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This file is part of 'miniver': https://github.com/jbweston/miniver
+
+import os
+
+# No public API
+__all__ = []
+
+package_root = os.path.dirname(os.path.realpath(__file__))
+package_name = os.path.basename(package_root)
+
+STATIC_VERSION_FILE = "_static_version.py"
+
+
+def get_version(version_file=STATIC_VERSION_FILE):
+ override = os.environ.get("SETUPTOOLS_SCM_PRETEND_VERSION")
+ if override is not None and override != "":
+ return override
+ version_info = get_static_version_info(version_file)
+ version = version_info["version"]
+ return version
+
+
+def get_static_version_info(version_file=STATIC_VERSION_FILE):
+ version_info = {}
+ with open(os.path.join(package_root, version_file), "rb") as f:
+ exec(f.read(), {}, version_info)
+ return version_info
+
+
+__version__ = get_version()
+
+if __name__ == "__main__":
+ print("Version: ", get_version())
diff --git a/python/tests/test_capsules.py b/python/tests/test_capsules.py
index f124dec6..aef037f9 100644
--- a/python/tests/test_capsules.py
+++ b/python/tests/test_capsules.py
@@ -14,11 +14,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import pyarrow as pa
+
import pytest
import nanoarrow as na
+pa = pytest.importorskip("pyarrow")
+
class SchemaWrapper:
def __init__(self, schema):
diff --git a/python/tests/test_device.py b/python/tests/test_device.py
index 1426e360..6ba6773c 100644
--- a/python/tests/test_device.py
+++ b/python/tests/test_device.py
@@ -15,10 +15,12 @@
# specific language governing permissions and limitations
# under the License.
-import pyarrow as pa
+import pytest
from nanoarrow import device
+pa = pytest.importorskip("pyarrow")
+
def test_cpu_device():
cpu = device.Device.cpu()
diff --git a/python/tests/test_nanoarrow.py b/python/tests/test_nanoarrow.py
index 816ff141..5d0d9840 100644
--- a/python/tests/test_nanoarrow.py
+++ b/python/tests/test_nanoarrow.py
@@ -15,19 +15,14 @@
# specific language governing permissions and limitations
# under the License.
-import re
import sys
-import numpy as np
-import pyarrow as pa
import pytest
import nanoarrow as na
-
-def test_c_version():
- re_version = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(-SNAPSHOT)?$")
- assert re_version.match(na.c_version()) is not None
+np = pytest.importorskip("numpy")
+pa = pytest.importorskip("pyarrow")
def test_schema_helper():
diff --git a/python/MANIFEST.in b/python/tests/test_version.py
similarity index 70%
copy from python/MANIFEST.in
copy to python/tests/test_version.py
index 9fc29372..2b080d64 100644
--- a/python/MANIFEST.in
+++ b/python/tests/test_version.py
@@ -15,7 +15,16 @@
# specific language governing permissions and limitations
# under the License.
-exclude bootstrap.py
-include src/nanoarrow/nanoarrow.c
-include src/nanoarrow/nanoarrow.h
-include src/nanoarrow/nanoarrow_c.pxd
+import re
+
+import nanoarrow as na
+
+
+def test_version():
+ re_py_version = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(dev[0-9+])?$")
+ assert re_py_version.match(na.__version__) is not None
+
+
+def test_c_version():
+ re_version = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(-SNAPSHOT)?$")
+ assert re_version.match(na.c_version()) is not None
