This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch release/0.5.x in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
commit c402f065c4d1b495eecee7192d0de87ad6964d24 Author: Y Ethan Guo <[email protected]> AuthorDate: Fri Sep 4 23:00:37 2026 -0700 fix(ci): install a modern protoc for the manylinux wheel build (#751) (cherry picked from commit 7d130d812850cf34348df9440d0e038a8f4f2f92) --- .github/scripts/manylinux-build-deps.sh | 111 ++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 43 +++++++++++++ .github/workflows/release.yml | 42 ++++++------ 3 files changed, 175 insertions(+), 21 deletions(-) diff --git a/.github/scripts/manylinux-build-deps.sh b/.github/scripts/manylinux-build-deps.sh new file mode 100755 index 00000000..3b313efb --- /dev/null +++ b/.github/scripts/manylinux-build-deps.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Install the system build dependencies for a manylinux wheel. +# +# Both release steps and the CI job that guards them run this one script, so a +# change cannot land in some of them and not others. +# +# protoc: the distro package is 2.5.0 on the CentOS 7 base, which predates the +# --experimental_allow_proto3_optional flag prost-build passes when it compiles +# lance-encoding/lance-file's .proto files. +# +# libclang: librocksdb-sys generates its bindings with bindgen, which needs a +# libclang the base image does not carry; the distro clang is 3.4, older than +# bindgen supports. bindgen also needs clang's builtin headers (stdbool.h and +# friends), which libclang cannot locate on its own when loaded from the SCL +# prefix, so both are handed over explicitly. The exports only reach the build +# if this script is SOURCED from before-script-linux, not executed: a step-level +# `env:` never enters the container (maturin-action forwards only its own +# variables), while the before-script runs in the same shell as the build - the +# long-standing CFLAGS_aarch64 export below relies on the same behavior. +# +# perl-IPC-Cmd: needed by openssl. + +# This script is sourced, not executed, so that the exports below land in the +# same shell the build runs in. Keep the caller's shell options: the strictness +# is for this script, and leaking `set -u` in particular would turn any unset +# variable in a future maturin-action release into a release-time failure. +_deps_saved_opts=$(set +o) +set -euo pipefail + +PROTOC_VERSION=36.1 + +case "$(uname -m)" in + x86_64) + protoc_arch=x86_64 + protoc_sha256=c4bc672d9d49214dc8cafdceadf4df92182d6ca8e3ec65a56b2d7de5602669b4 + ;; + aarch64) + protoc_arch=aarch_64 + protoc_sha256=237a68856edf1bd28b6204bddd0596c1cf46d298bc29c620012540b2e44c73e7 + ;; + *) + echo "unsupported architecture: $(uname -m)" >&2 + exit 1 + ;; +esac + +curl -fsSL -o /tmp/protoc.zip \ + "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${protoc_arch}.zip"; + +# The wheels this produces are published and signed, so verify the archive +# rather than trusting whatever the download returned. +echo "${protoc_sha256} /tmp/protoc.zip" | sha256sum -c - + +unzip -qo /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' +chmod +x /usr/local/bin/protoc +protoc --version + +yum install -y llvm-toolset-7.0-clang llvm-toolset-7.0-clang-libs perl-IPC-Cmd + +# libclang.so links against the LLVM runtime in the same prefix, so make the +# loader aware of it rather than relying on a symlink out of the prefix. +libclang_dir=/opt/rh/llvm-toolset-7.0/root/usr/lib64 +if [ ! -e "$libclang_dir/libclang.so" ]; then + echo "libclang.so not found under $libclang_dir" >&2 + return 1 2>/dev/null || exit 1 +fi + +# Register the prefix so the LLVM runtime libclang links against resolves. +echo "$libclang_dir" >/etc/ld.so.conf.d/llvm-toolset-7.0.conf +ldconfig + +# Assign in a condition context: a bare `var=$(...)` takes the substitution's +# status, so a non-matching glob would trip errexit before the check below. +clang_include="" +for candidate in "$libclang_dir"/clang/*/include; do + if [ -e "$candidate/stdbool.h" ]; then + clang_include=$candidate + break + fi +done +if [ -z "$clang_include" ]; then + echo "clang builtin headers not found under $libclang_dir/clang" >&2 + return 1 2>/dev/null || exit 1 +fi + +export LIBCLANG_PATH="$libclang_dir" +export BINDGEN_EXTRA_CLANG_ARGS="-I$clang_include" +echo "libclang: $LIBCLANG_PATH" +echo "clang builtin headers: $clang_include" + +# Restore the caller's shell options; the exports above survive. +eval "$_deps_saved_opts" +unset _deps_saved_opts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25f11af2..9ce83fd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -307,6 +307,49 @@ jobs: run: | sudo chown -R $(id -u):$(id -g) $GITHUB_WORKSPACE/.cargo $GITHUB_WORKSPACE/target $GITHUB_WORKSPACE/python/target $GITHUB_WORKSPACE/.uv-cache || true + manylinux-wheel-build: + name: Build the manylinux wheel + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-latest + target: x86_64-unknown-linux-gnu + - runner: ubuntu-22.04-arm + target: aarch64-unknown-linux-gnu + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + # Builds the wheel in the same container, running the same dependency + # script, that the release workflow publishes from. Regular CI runs on the + # host, whose apt protoc is new enough for lance-encoding's prost-build and + # which carries libclang for librocksdb-sys, so a container-only break (see + # #750) stays invisible until a release tag is pushed - by which point the + # version is burnt on crates.io and cannot be reused. + - name: Build wheel + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + command: build + args: --release -m python/Cargo.toml --out dist + # Pin the image rather than relying on the default, so this job and the + # release steps are provably the same environment. + manylinux: '2014' + before-script-linux: | + source .github/scripts/manylinux-build-deps.sh + # A wheel that compiles is not necessarily a wheel that loads: the + # bindings rocksdb generates come from the container's older libclang, and + # bad bindings tend to surface on import or first use rather than at + # compile time. Each runner installs the wheel it just built. + - name: Import the built wheel + run: | + python -m pip install --upgrade pip + python -m pip install dist/*.whl + python -c "import hudi; print(hudi.__file__)" + publish-coverage: name: Publish coverage reports to codecov.io runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 653ae04f..eeaf35d2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -119,37 +119,37 @@ jobs: release-pypi-manylinux: name: PyPI release manylinux needs: validate-release-tag - runs-on: ubuntu-latest + # aarch64 builds on a native ARM runner so this job and the CI wheel-build + # job use the same image; cross-building it from x86_64 would exercise a + # container CI never touches. It also drops the ring workaround that the + # cross build needed (briansmith/ring#1728). + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-latest + target: x86_64-unknown-linux-gnu + # the sdist rides along with exactly one leg + args: --skip-existing -m python/Cargo.toml + - runner: ubuntu-22.04-arm + target: aarch64-unknown-linux-gnu + args: --skip-existing -m python/Cargo.toml --no-sdist + runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: python-version: '3.13' - - name: Publish manylinux to pypi x86_64 (with sdist) + - name: Publish manylinux to pypi uses: PyO3/maturin-action@v1 env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} MATURIN_REPOSITORY: pypi with: - target: x86_64-unknown-linux-gnu - command: publish - args: --skip-existing -m python/Cargo.toml - # protobuf-compiler is needed by lance-encoding/lance-file's prost-build; - # perl-IPC-Cmd is needed by openssl. - before-script-linux: yum install -y protobuf-compiler perl-IPC-Cmd - - - name: Publish manylinux to pypi aarch64 (without sdist) - uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - MATURIN_REPOSITORY: pypi - with: - target: aarch64-unknown-linux-gnu + target: ${{ matrix.target }} command: publish - args: --skip-existing -m python/Cargo.toml --no-sdist + args: ${{ matrix.args }} + manylinux: '2014' before-script-linux: | - yum install -y protobuf-compiler - # We can remove this once we upgrade to 2_28. - # https://github.com/briansmith/ring/issues/1728 - export CFLAGS_aarch64_unknown_linux_gnu="-D__ARM_ARCH=8" + source .github/scripts/manylinux-build-deps.sh
