This is an automated email from the ASF dual-hosted git repository.
jorisvandenbossche pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new a1c1773b72 GH-39555: [Packaging][Python] Enable building pyarrow
against numpy 2.0 (#39557)
a1c1773b72 is described below
commit a1c1773b724e4d78faf9a097247c7e976cd2cbfa
Author: Joris Van den Bossche <[email protected]>
AuthorDate: Thu Feb 1 14:53:35 2024 +0100
GH-39555: [Packaging][Python] Enable building pyarrow against numpy 2.0
(#39557)
### Rationale for this change
Ensure we can build pyarrow against numpy 2.0 nightly (update
pyproject.toml to allow this), and test this by building our nightly wheels
with numpy nightly. This also ensures that other projects that use our nightly
wheels to test together with numpy nightly can do that (numpy 2.0 changes the
ABI, so to run with numpy 2.0, your package needs to be built with numpy 2.x;
currently pyarrow installed with our nightly wheel will fail to import when
also numpy nightly is installed).
See the parent issue https://github.com/apache/arrow/issues/39532 for
details, and
https://numpy.org/devdocs/dev/depending_on_numpy.html#numpy-2-0-specific-advice
for a direct link to the NumPy guidelines on updating build dependencies for
NumPy 2.0.
* Closes: #39555
Lead-authored-by: Joris Van den Bossche <[email protected]>
Co-authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Joris Van den Bossche <[email protected]>
---
ci/docker/python-wheel-manylinux.dockerfile | 5 +++--
ci/docker/python-wheel-windows-vs2017.dockerfile | 3 ++-
ci/scripts/python_wheel_macos_build.sh | 5 ++++-
python/pyproject.toml | 7 ++++++-
python/requirements-build.txt | 3 ++-
python/requirements-wheel-build.txt | 3 ++-
python/setup.py | 2 +-
7 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/ci/docker/python-wheel-manylinux.dockerfile
b/ci/docker/python-wheel-manylinux.dockerfile
index 0a50d450c2..a07c727ac7 100644
--- a/ci/docker/python-wheel-manylinux.dockerfile
+++ b/ci/docker/python-wheel-manylinux.dockerfile
@@ -28,7 +28,7 @@ ENV MANYLINUX_VERSION=${manylinux}
RUN yum install -y dnf
# Install basic dependencies
-RUN dnf install -y git flex curl autoconf zip perl-IPC-Cmd wget kernel-headers
+RUN dnf install -y git flex curl autoconf zip perl-IPC-Cmd wget
# A system Python is required for ninja and vcpkg in this Dockerfile.
# On manylinux2014 base images, system Python is 2.7.5, while
@@ -97,4 +97,5 @@ SHELL ["/bin/bash", "-i", "-c"]
ENTRYPOINT ["/bin/bash", "-i", "-c"]
COPY python/requirements-wheel-build.txt /arrow/python/
-RUN pip install -r /arrow/python/requirements-wheel-build.txt
+# TODO(GH-39848) Remove the `--pre --extra-index-url` for numpy nightly again
before the 16.0 release
+RUN pip install -r /arrow/python/requirements-wheel-build.txt --pre
--extra-index-url
"https://pypi.anaconda.org/scientific-python-nightly-wheels/simple";
diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile
b/ci/docker/python-wheel-windows-vs2017.dockerfile
index faf07800c9..067105b3a7 100644
--- a/ci/docker/python-wheel-windows-vs2017.dockerfile
+++ b/ci/docker/python-wheel-windows-vs2017.dockerfile
@@ -88,7 +88,8 @@ RUN choco install -r -y --no-progress python
--version=%PYTHON_VERSION%
RUN python -m pip install -U pip setuptools
COPY python/requirements-wheel-build.txt arrow/python/
-RUN python -m pip install -r arrow/python/requirements-wheel-build.txt
+# TODO(GH-39848) Remove the `--pre --extra-index-url` for numpy nightly again
before the 16.0 release
+RUN python -m pip install -r arrow/python/requirements-wheel-build.txt --pre
--extra-index-url
"https://pypi.anaconda.org/scientific-python-nightly-wheels/simple";
# ENV CLCACHE_DIR="C:\clcache"
# ENV CLCACHE_COMPRESS=1
diff --git a/ci/scripts/python_wheel_macos_build.sh
b/ci/scripts/python_wheel_macos_build.sh
index fd845c512d..8123a9fdf1 100755
--- a/ci/scripts/python_wheel_macos_build.sh
+++ b/ci/scripts/python_wheel_macos_build.sh
@@ -50,12 +50,15 @@ echo "=== (${PYTHON_VERSION}) Install Python build
dependencies ==="
export PIP_SITE_PACKAGES=$(python -c 'import site;
print(site.getsitepackages()[0])')
export PIP_TARGET_PLATFORM="macosx_${MACOSX_DEPLOYMENT_TARGET//./_}_${arch}"
+# TODO(GH-39848) Remove the `--pre --extra-index-url` for numpy nightly again
before the 16.0 release
pip install \
--upgrade \
--only-binary=:all: \
--target $PIP_SITE_PACKAGES \
--platform $PIP_TARGET_PLATFORM \
- -r ${source_dir}/python/requirements-wheel-build.txt
+ -r ${source_dir}/python/requirements-wheel-build.txt \
+ --pre \
+ --extra-index-url
"https://pypi.anaconda.org/scientific-python-nightly-wheels/simple";
pip install "delocate>=0.10.3"
echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 437de105ab..9079618ad1 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -18,7 +18,12 @@
[build-system]
requires = [
"cython >= 0.29.31",
- "oldest-supported-numpy>=0.14",
+ # Starting with NumPy 1.25, NumPy is (by default) as far back compatible
+ # as oldest-support-numpy was (customizable with a NPY_TARGET_VERSION
+ # define). For older Python versions (where NumPy 1.25 is not yet
avaiable)
+ # continue using oldest-support-numpy.
+ "oldest-supported-numpy>=0.14; python_version<'3.9'",
+ "numpy>=1.25; python_version>='3.9'",
"setuptools_scm < 8.0.0",
"setuptools >= 40.1.0",
"wheel"
diff --git a/python/requirements-build.txt b/python/requirements-build.txt
index 56e9d479ee..e1372e807f 100644
--- a/python/requirements-build.txt
+++ b/python/requirements-build.txt
@@ -1,4 +1,5 @@
cython>=0.29.31
-oldest-supported-numpy>=0.14
+oldest-supported-numpy>=0.14; python_version<'3.9'
+numpy>=1.25; python_version>='3.9'
setuptools_scm<8.0.0
setuptools>=38.6.0
diff --git a/python/requirements-wheel-build.txt
b/python/requirements-wheel-build.txt
index f42ee4a018..044f9de5f8 100644
--- a/python/requirements-wheel-build.txt
+++ b/python/requirements-wheel-build.txt
@@ -1,5 +1,6 @@
cython>=0.29.31
-oldest-supported-numpy>=0.14
+oldest-supported-numpy>=0.14; python_version<'3.9'
+numpy>=1.25; python_version>='3.9'
setuptools_scm<8.0.0
setuptools>=58
wheel
diff --git a/python/setup.py b/python/setup.py
index d7a2da2077..098d75a318 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -449,7 +449,7 @@ class BinaryDistribution(Distribution):
install_requires = (
- 'numpy >= 1.16.6, <2',
+ 'numpy >= 1.16.6',
)
