This is an automated email from the ASF dual-hosted git repository. raulcd pushed a commit to branch maint-14.0.0 in repository https://gitbox.apache.org/repos/asf/arrow.git
commit 4db1888787c5d7f249dd9db1e1033e320c751880 Author: Raúl Cumplido <[email protected]> AuthorDate: Wed Oct 11 16:33:18 2023 +0200 GH-37880: [CI][Python][Packaging] Add support for Python 3.12 (#37901) ### Rationale for this change Python 3.12 will be released in the next couple of weeks. We should add the wheels for pyarrow on our 14.0.0 release. ### What changes are included in this PR? This PR adds jobs to build pyarrow wheels for Python 3.12. ### Are these changes tested? They will be tested via archery tasks ### Are there any user-facing changes? No but users will be able to use pyarrow with Python 3.12 * Closes: #37880 Authored-by: Raúl Cumplido <[email protected]> Signed-off-by: Raúl Cumplido <[email protected]> --- .github/workflows/cpp.yml | 8 ++++++++ .github/workflows/java_jni.yml | 2 +- ci/docker/linux-apt-lint.dockerfile | 6 +----- ci/docker/python-wheel-windows-test-vs2017.dockerfile | 5 +++-- ci/docker/python-wheel-windows-vs2017.dockerfile | 5 +++-- ci/scripts/install_gcs_testbench.sh | 4 +++- ci/scripts/install_python.sh | 7 ++++--- ci/vcpkg/ports.patch | 13 +++++++++++++ dev/release/verify-release-candidate.sh | 6 +++--- dev/tasks/python-wheels/github.osx.arm64.yml | 4 ++-- dev/tasks/tasks.yml | 7 ++++--- docker-compose.yml | 7 ++++--- python/requirements-wheel-test.txt | 7 ++----- python/setup.py | 1 + 14 files changed, 52 insertions(+), 30 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 5451cbe064..a9361f9f51 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -204,6 +204,10 @@ jobs: run: | $(brew --prefix bash)/bin/bash \ ci/scripts/install_minio.sh latest /usr/local + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 - name: Install Google Cloud Storage Testbench run: ci/scripts/install_gcs_testbench.sh default - name: Install Azurite Storage Emulator @@ -427,6 +431,10 @@ jobs: --output-document /usr/local/bin/minio.exe \ https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2022-05-26T05-48-41Z chmod +x /usr/local/bin/minio.exe + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 - name: Install Google Cloud Storage Testbench shell: bash run: | diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml index 4ebfccbe84..467e8a88af 100644 --- a/.github/workflows/java_jni.yml +++ b/.github/workflows/java_jni.yml @@ -53,7 +53,7 @@ jobs: name: AMD64 manylinux2014 Java JNI runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 + timeout-minutes: 500 steps: - name: Checkout Arrow uses: actions/checkout@v4 diff --git a/ci/docker/linux-apt-lint.dockerfile b/ci/docker/linux-apt-lint.dockerfile index 8a679be2eb..2b94a48871 100644 --- a/ci/docker/linux-apt-lint.dockerfile +++ b/ci/docker/linux-apt-lint.dockerfile @@ -67,11 +67,7 @@ COPY ci/scripts/install_iwyu.sh /arrow/ci/scripts/ RUN arrow/ci/scripts/install_iwyu.sh /tmp/iwyu /usr/local ${clang_tools} # Use python3 by default in scripts -RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ - ln -s /usr/bin/pip3 /usr/local/bin/pip - -COPY dev/archery/setup.py /arrow/dev/archery/ -RUN pip install -e arrow/dev/archery[lint] +RUN ln -s /usr/bin/python3 /usr/local/bin/python ENV LC_ALL=C.UTF-8 \ LANG=C.UTF-8 diff --git a/ci/docker/python-wheel-windows-test-vs2017.dockerfile b/ci/docker/python-wheel-windows-test-vs2017.dockerfile index 4aa5363ff7..e842ede184 100644 --- a/ci/docker/python-wheel-windows-test-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-test-vs2017.dockerfile @@ -37,7 +37,8 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ ARG python=3.8 RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ - (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ - (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") + (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ + (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ + (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index 531c4e678f..b6a8125a5c 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -81,8 +81,9 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ ARG python=3.8 RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \ - (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ - (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") + (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \ + (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts") & \ + (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0" && setx PATH "%PATH%;C:\Python312;C:\Python312\Scripts") RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% RUN python -m pip install -U pip setuptools diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh index 26fca3c946..0aa6d20975 100755 --- a/ci/scripts/install_gcs_testbench.sh +++ b/ci/scripts/install_gcs_testbench.sh @@ -36,7 +36,9 @@ esac version=$1 if [[ "${version}" -eq "default" ]]; then - version="v0.36.0" + version="v0.39.0" + # Latests versions of Testbench require newer setuptools + ${PYTHON:-python3} -m pip install --upgrade setuptools fi ${PYTHON:-python3} -m pip install \ diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index 599c238c97..8487090f7a 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -27,8 +27,9 @@ platforms=([windows]=Windows declare -A versions versions=([3.8]=3.8.10 [3.9]=3.9.13 - [3.10]=3.10.8 - [3.11]=3.11.0) + [3.10]=3.10.11 + [3.11]=3.11.5 + [3.12]=3.12.0) if [ "$#" -ne 2 ]; then echo "Usage: $0 <platform> <version>" @@ -45,7 +46,7 @@ full_version=${versions[$2]} if [ $platform = "MacOSX" ]; then echo "Downloading Python installer..." - if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ]; then + if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ] || [ "$version" = "3.12" ]; then fname="python-${full_version}-macos11.pkg" else fname="python-${full_version}-macosx10.9.pkg" diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch index 45feb3ba0e..68f6cae5ad 100644 --- a/ci/vcpkg/ports.patch +++ b/ci/vcpkg/ports.patch @@ -51,3 +51,16 @@ index 0000000..a57ce0c + } + + static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) { +diff --git a/ports/llvm/portfile.cmake b/ports/llvm/portfile.cmake +index 4d7e26a..1f054a2 100644 +--- a/ports/llvm/portfile.cmake ++++ b/ports/llvm/portfile.cmake +@@ -274,6 +274,8 @@ vcpkg_cmake_configure( + COMPILER_RT_ENABLE_IOS + OPENMP_TOOLS_INSTALL_DIR + MLIR_TOOLS_INSTALL_DIR ++ BOLT_TOOLS_INSTALL_DIR ++ LIBOMP_INSTALL_ALIASES + ) + + vcpkg_cmake_install(ADD_BIN_TO_PATH) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 2e5aa1c393..6397cd4c32 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -1052,7 +1052,7 @@ test_linux_wheels() { local arch="x86_64" fi - local python_versions="${TEST_PYTHON_VERSIONS:-3.8 3.9 3.10 3.11}" + local python_versions="${TEST_PYTHON_VERSIONS:-3.8 3.9 3.10 3.11 3.12}" local platform_tags="${TEST_WHEEL_PLATFORM_TAGS:-manylinux_2_17_${arch}.manylinux2014_${arch} manylinux_2_28_${arch}}" for python in ${python_versions}; do @@ -1074,11 +1074,11 @@ test_macos_wheels() { # apple silicon processor if [ "$(uname -m)" = "arm64" ]; then - local python_versions="3.8 3.9 3.10 3.11" + local python_versions="3.8 3.9 3.10 3.11 3.12" local platform_tags="macosx_11_0_arm64" local check_flight=OFF else - local python_versions="3.8 3.9 3.10 3.11" + local python_versions="3.8 3.9 3.10 3.11 3.12" local platform_tags="macosx_10_14_x86_64" fi diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml index 2c796d162d..c217f5901b 100644 --- a/dev/tasks/python-wheels/github.osx.arm64.yml +++ b/dev/tasks/python-wheels/github.osx.arm64.yml @@ -104,9 +104,9 @@ jobs: # the cffi wheel. We build cffi with the flags pointing to the correct libffi location. LDFLAGS=-L$(brew --prefix libffi)/lib CFLAGS=-I$(brew --prefix libffi)/include \ pip install cffi --no-binary :all: - pip install -r arrow/python/requirements-wheel-test.txt + pip install -r arrow/python/requirements-wheel-test.txt PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default - arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow + arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow - name: Upload artifacts shell: bash diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index e32e61bfbc..0950378875 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -467,7 +467,8 @@ tasks: {% for python_version, python_tag, abi_tag in [("3.8", "cp38", "cp38"), ("3.9", "cp39", "cp39"), ("3.10", "cp310", "cp310"), - ("3.11", "cp311", "cp311")] %} + ("3.11", "cp311", "cp311"), + ("3.12", "cp312", "cp312")] %} {############################## Wheel Linux ##################################} @@ -533,7 +534,7 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-macosx_11_0_arm64.whl -{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310"), ("3.11", "cp311")] %} +{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310"), ("3.11", "cp311"), ("3.12", "cp312")] %} wheel-macos-big-sur-{{ python_tag }}-arm64: ci: github template: python-wheels/github.osx.arm64.yml @@ -1233,7 +1234,7 @@ tasks: flags: "-e ARROW_CSV=ON -e ARROW_PARQUET=ON" image: ubuntu-cpp-minimal -{% for python_version in ["3.8", "3.9", "3.10", "3.11"] %} +{% for python_version in ["3.8", "3.9", "3.10", "3.11", "3.12"] %} test-conda-python-{{ python_version }}: ci: github template: docker-tests/github.linux.yml diff --git a/docker-compose.yml b/docker-compose.yml index 62e5aee0a8..10e2b9fa8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1033,7 +1033,7 @@ services: args: arch: ${ARCH} arch_short: ${ARCH_SHORT} - base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2022-10-25-fbea779 + base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2023-10-03-72cdc42 vcpkg: ${VCPKG} python: ${PYTHON} manylinux: 2014 @@ -1056,7 +1056,7 @@ services: args: arch: ${ARCH} arch_short: ${ARCH_SHORT} - base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2023-01-14-103cb93 + base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2023-10-03-72cdc42 vcpkg: ${VCPKG} python: ${PYTHON} manylinux: 2_28 @@ -1807,7 +1807,8 @@ services: command: > /bin/bash -c " git config --global --add safe.directory /arrow && - archery lint --all --no-clang-tidy --no-iwyu --no-numpydoc" + pip install arrow/dev/archery[lint] && + archery lint --all --no-clang-tidy --no-iwyu --no-numpydoc --src /arrow" ######################### Integration Tests ################################# diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index 9de0acb754..516ec0fccc 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -7,18 +7,15 @@ pytz tzdata; sys_platform == 'win32' numpy==1.21.3; platform_system == "Linux" and platform_machine == "aarch64" and python_version < "3.11" -numpy==1.23.4; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.11" +numpy==1.23.4; python_version == "3.11" +numpy==1.26.0; python_version >= "3.12" numpy==1.19.5; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.9" numpy==1.21.3; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.9" and python_version < "3.11" -numpy==1.23.4; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.11" numpy==1.21.3; platform_system == "Darwin" and platform_machine == "arm64" and python_version < "3.11" -numpy==1.23.4; platform_system == "Darwin" and platform_machine == "arm64" and python_version >= "3.11" numpy==1.19.5; platform_system == "Darwin" and platform_machine != "arm64" and python_version < "3.9" numpy==1.21.3; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.9" and python_version < "3.11" -numpy==1.23.4; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.11" numpy==1.19.5; platform_system == "Windows" and python_version < "3.9" numpy==1.21.3; platform_system == "Windows" and python_version >= "3.9" and python_version < "3.11" -numpy==1.23.4; platform_system == "Windows" and python_version >= "3.11" pandas<1.1.0; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.8" pandas; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.8" diff --git a/python/setup.py b/python/setup.py index 6934161fe6..3686dc4573 100755 --- a/python/setup.py +++ b/python/setup.py @@ -505,6 +505,7 @@ setup( 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ], license='Apache License, Version 2.0', maintainer='Apache Arrow Developers',
