This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new b3c952a3 chore(ci): Integrate Python into more CI workflows (#359)
b3c952a3 is described below
commit b3c952a3e21c2b47df85dbede3444f852614a3e2
Author: Dewey Dunnington <[email protected]>
AuthorDate: Wed Jan 17 09:32:49 2024 -0400
chore(ci): Integrate Python into more CI workflows (#359)
This PR adds Python to the coverage and verify workflows (except on
centos7, where the image isn't set up for a supported version of Python
yet). There were also some housekeeping issues identified by various
logs that I noticed in the process (e.g., fixing the Python dev version
format, ensuring that CMAKE_BIN is respected in bootstrap.py).
---
.github/workflows/coverage.yaml | 4 +--
.github/workflows/python.yaml | 25 ++------------
.github/workflows/verify.yaml | 4 ++-
ci/scripts/coverage.sh | 38 +++++++++++++++++++++
dev/release/verify-release-candidate.sh | 58 +++++++++++++++++++++++++++++++++
python/bootstrap.py | 9 +++--
python/pyproject.toml | 8 +++--
python/setup.py | 3 +-
python/src/nanoarrow/_lib.pyx | 2 +-
python/src/nanoarrow/_static_version.py | 2 +-
python/tests/test_version.py | 2 +-
11 files changed, 119 insertions(+), 36 deletions(-)
diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml
index 25dd1cf4..74876381 100644
--- a/.github/workflows/coverage.yaml
+++ b/.github/workflows/coverage.yaml
@@ -55,6 +55,6 @@ jobs:
path: _coverage
- name: Upload coverage to codecov
- uses: codecov/codecov-action@v2
+ uses: codecov/codecov-action@v3
with:
- files: '_coverage/coverage.info,_coverage/r_coverage.json'
+ files:
'_coverage/coverage.info,_coverage/r_coverage.json,_coverage/python_coverage.xml'
diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml
index 85f12c2e..c60f2290 100644
--- a/.github/workflows/python.yaml
+++ b/.github/workflows/python.yaml
@@ -36,7 +36,7 @@ jobs:
strategy:
matrix:
os: [ubuntu-latest]
- python-version: ['3.10']
+ python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
steps:
- uses: actions/checkout@v3
@@ -59,27 +59,6 @@ jobs:
pytest python/tests -v -s
- name: Run doctests
- if: success() && matrix.python-version == '3.10'
+ if: success() && matrix.python-version == '3.12'
run: |
pytest --pyargs nanoarrow --doctest-modules
-
- - name: Coverage
- if: success() && matrix.python-version == '3.10'
- run: |
- pip uninstall --yes nanoarrow
- pip install pytest-cov Cython
- pushd python
-
- # Build with Cython + gcc coverage options
- pip install -e .
- NANOARROW_PYTHON_COVERAGE=1 python setup.py build_ext --inplace
-
- # Run tests + coverage.py (generates .coverage + coverage.xml files)
- python -m pytest --cov ./src/nanoarrow
- python -m coverage xml
-
- - name: Upload coverage to codecov
- if: success() && matrix.python-version == '3.10'
- uses: codecov/codecov-action@v2
- with:
- files: 'python/coverage.xml'
diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml
index c5d12360..b0943905 100644
--- a/.github/workflows/verify.yaml
+++ b/.github/workflows/verify.yaml
@@ -138,7 +138,9 @@ jobs:
- {
platform: "centos7",
arch: "amd64",
- compose_args: "-e NANOARROW_ACCEPT_IMPORT_GPG_KEYS_ERROR=1"
+ # Currently the Python on the centos7 image is 3.6, which does
not support
+ # new enough setuptools to build the Python package.
+ compose_args: "-e NANOARROW_ACCEPT_IMPORT_GPG_KEYS_ERROR=1 -e
TEST_PYTHON=0"
}
- {
platform: "ubuntu",
diff --git a/ci/scripts/coverage.sh b/ci/scripts/coverage.sh
index ad7374ab..8c866bb9 100755
--- a/ci/scripts/coverage.sh
+++ b/ci/scripts/coverage.sh
@@ -52,7 +52,15 @@ case $# in
;;
esac
+maybe_activate_venv() {
+ if [ ! -z "${NANOARROW_PYTHON_VENV}" ]; then
+ source "${NANOARROW_PYTHON_VENV}/bin/activate"
+ fi
+}
+
function main() {
+ maybe_activate_venv
+
SANDBOX_DIR="${TARGET_NANOARROW_DIR}/_coverage"
if [ -d "${SANDBOX_DIR}" ]; then
rm -rf "${SANDBOX_DIR}"
@@ -107,6 +115,7 @@ function main() {
--exclude "*/gtest/*" \
--exclude "*/flatcc/*" \
--exclude "*_generated.h" \
+ --exclude "*nanoarrow/_deps/*" \
--output-file coverage.info
# Generate the html coverage while we're here
@@ -137,6 +146,35 @@ function main() {
show_header "R package coverage summary"
Rscript -e 'library(covr); print(readRDS("r_coverage.rds"))'
popd
+
+ # Build + test Python package with cython/gcc coverage options
+ show_header "Build + test Python package"
+ pushd "${SANDBOX_DIR}"
+ TARGET_NANOARROW_PYTHON_DIR="${TARGET_NANOARROW_DIR}/python"
+
+ pushd "${TARGET_NANOARROW_PYTHON_DIR}"
+ NANOARROW_PYTHON_COVERAGE=1 python -m pip install -e .
+
+ # Run tests + coverage.py (generates .coverage with absolute file paths)
+ python -m pytest --cov ./src/nanoarrow
+
+ # Generate HTML report (file paths not important since it's just for
viewing)
+ python -m coverage html
+ mv htmlcov "${SANDBOX_DIR}/python_htmlcov"
+
+ # Move .coverage to the root directory and generate coverage.xml
+ # (generates relative file paths from the root of the repo)
+ mv .coverage ..
+ cp .coveragerc ..
+ pushd ..
+ python -m coverage xml
+ mv coverage.xml "${SANDBOX_DIR}/python_coverage.xml"
+ mv .coverage "${SANDBOX_DIR}/python_coverage.db"
+ rm .coveragerc
+ popd
+
+ popd
+ popd
}
main
diff --git a/dev/release/verify-release-candidate.sh
b/dev/release/verify-release-candidate.sh
index 250bbe64..84013602 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -297,6 +297,59 @@ test_r() {
popd
}
+activate_or_create_venv() {
+ if [ ! -z "${NANOARROW_PYTHON_VENV}" ]; then
+ show_info "Activating virtual environment at ${NANOARROW_PYTHON_VENV}"
+ # bash on Windows needs venv/Scripts/activate instead of venv/bin/activate
+ source "${NANOARROW_PYTHON_VENV}/bin/activate" || source
"${NANOARROW_PYTHON_VENV}/Scripts/activate"
+ else
+ # Try python3 first, then try regular python (e.g., Windows)
+ if [ -z "${PYTHON_BIN}" ] && python3 --version >/dev/null; then
+ PYTHON_BIN=python3
+ elif [ -z "${PYTHON_BIN}" ]; then
+ PYTHON_BIN=python
+ fi
+
+ show_info "Creating temporary virtual environment using ${PYTHON_BIN}..."
+ "${PYTHON_BIN}" -m venv "${NANOARROW_TMPDIR}/venv"
+ # bash on Windows needs venv/Scripts/activate instead of venv/bin/activate
+ source "${NANOARROW_TMPDIR}/venv/bin/activate" || source
"${NANOARROW_TMPDIR}/venv/Scripts/activate"
+ python -m pip install --upgrade pip
+ fi
+}
+
+test_python() {
+ show_header "Build and test Python package"
+ activate_or_create_venv
+
+ show_info "Installing build utilities"
+ python -m pip install --upgrade build
+
+ pushd "${NANOARROW_SOURCE_DIR}/python"
+
+ show_info "Building Python package"
+ rm -rf "${NANOARROW_TMPDIR}/python"
+ python -m build --wheel --outdir "${NANOARROW_TMPDIR}/python"
+ PYTHON_WHEEL_NAME=$(ls "${NANOARROW_TMPDIR}/python" | grep -e ".whl")
+
+ # On Windows bash, pip install needs a Windows-style path
+ if uname | grep -e "_NT-" >/dev/null; then
+ pushd "${NANOARROW_TMPDIR}"
+ PYTHON_WHEEL_PATH="$(pwd -W)/python/${PYTHON_WHEEL_NAME}"
+ popd
+ else
+ PYTHON_WHEEL_PATH="${NANOARROW_TMPDIR}/python/${PYTHON_WHEEL_NAME}"
+ fi
+
+ show_info "Installing Python package"
+ python -m pip install --force-reinstall "${PYTHON_WHEEL_PATH}[verify]"
+
+ show_info "Testing wheel"
+ python -m pytest -vv
+
+ popd
+}
+
ensure_source_directory() {
show_header "Ensuring source directory"
@@ -346,6 +399,10 @@ test_source_distribution() {
test_r
fi
+ if [ ${TEST_PYTHON} -gt 0 ]; then
+ test_python
+ fi
+
popd
}
@@ -359,6 +416,7 @@ test_source_distribution() {
: ${TEST_C:=${TEST_SOURCE}}
: ${TEST_C_BUNDLED:=${TEST_C}}
: ${TEST_R:=${TEST_SOURCE}}
+: ${TEST_PYTHON:=${TEST_SOURCE}}
TEST_SUCCESS=no
diff --git a/python/bootstrap.py b/python/bootstrap.py
index b540058a..4fb2b0e9 100644
--- a/python/bootstrap.py
+++ b/python/bootstrap.py
@@ -186,7 +186,10 @@ def copy_or_generate_nanoarrow_c():
is_in_nanoarrow_repo = "nanoarrow.h" in os.listdir(
os.path.join(source_dir, "src", "nanoarrow")
)
- has_cmake = os.system("cmake --version") == 0
+ cmake_bin = os.getenv("CMAKE_BIN")
+ if not cmake_bin:
+ cmake_bin = "cmake"
+ has_cmake = os.system(f"{cmake_bin} --version") == 0
with tempfile.TemporaryDirectory() as build_dir:
if is_in_nanoarrow_repo:
@@ -206,7 +209,7 @@ def copy_or_generate_nanoarrow_c():
try:
subprocess.run(
[
- "cmake",
+ cmake_bin,
"-B",
build_dir,
"-S",
@@ -217,7 +220,7 @@ def copy_or_generate_nanoarrow_c():
)
subprocess.run(
[
- "cmake",
+ cmake_bin,
"--install",
build_dir,
"--prefix",
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 7850a09f..e176be26 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -22,15 +22,19 @@ dynamic = ["version"]
readme = "README.md"
description = "Python bindings to the nanoarrow C library"
authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
+maintainers = [{name = "Apache Arrow Developers", email =
"[email protected]"}]
license = {text = "Apache-2.0"}
requires-python = ">=3.8"
[project.optional-dependencies]
test = ["pyarrow", "pytest", "numpy"]
+verify = ["pytest", "numpy"]
[project.urls]
-homepage = "https://arrow.apache.org";
-repository = "https://github.com/apache/arrow-nanoarrow";
+Homepage = "https://arrow.apache.org";
+Repository = "https://github.com/apache/arrow-nanoarrow";
+Issues = "https://github.com/apache/arrow-nanoarrow/issues";
+Changelog = "https://github.com/apache/arrow-nanoarrow/blob/main/CHANGELOG.md";
[build-system]
requires = [
diff --git a/python/setup.py b/python/setup.py
index 92f8abc9..cdffda2b 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -52,7 +52,7 @@ if os.path.exists(bootstrap_py):
# Set some extra flags for compiling with coverage support
-if os.getenv("NANOARROW_COVERAGE") == "1":
+if os.getenv("NANOARROW_PYTHON_COVERAGE") == "1":
extra_compile_args = ["--coverage"]
extra_link_args = ["--coverage"]
extra_define_macros = [("CYTHON_TRACE", 1)]
@@ -65,7 +65,6 @@ else:
extra_link_args = []
extra_define_macros = []
-
setup(
ext_modules=[
Extension(
diff --git a/python/src/nanoarrow/_lib.pyx b/python/src/nanoarrow/_lib.pyx
index f77067c9..54d7bbdc 100644
--- a/python/src/nanoarrow/_lib.pyx
+++ b/python/src/nanoarrow/_lib.pyx
@@ -915,7 +915,7 @@ cdef class CBufferView:
if format_const != NULL:
snprintf(self._format, sizeof(self._format), "%s", format_const)
else:
- snprintf(self._format, sizeof(self._format), "%ds",
self._element_size_bits // 8)
+ snprintf(self._format, sizeof(self._format), "%ds",
<int>(self._element_size_bits // 8))
def __getbuffer__(self, Py_buffer *buffer, int flags):
if self._device.device_type != ARROW_DEVICE_CPU:
diff --git a/python/src/nanoarrow/_static_version.py
b/python/src/nanoarrow/_static_version.py
index 53569eff..33ecbc5c 100644
--- a/python/src/nanoarrow/_static_version.py
+++ b/python/src/nanoarrow/_static_version.py
@@ -18,7 +18,7 @@
# This file is part of 'miniver': https://github.com/jbweston/miniver
# Replaced by version-bumping scripts at release time
-version = "0.4.0dev0"
+version = "0.4.0.dev0"
# These values are only set if the distribution was created with 'git archive'
refnames = "$Format:%D$"
diff --git a/python/tests/test_version.py b/python/tests/test_version.py
index 2b080d64..701019cc 100644
--- a/python/tests/test_version.py
+++ b/python/tests/test_version.py
@@ -21,7 +21,7 @@ import nanoarrow as na
def test_version():
- re_py_version = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(dev[0-9+])?$")
+ re_py_version = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9+])?$")
assert re_py_version.match(na.__version__) is not None
