This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 984dfc89f feat(python): parallel python wheel build (#2989)
984dfc89f is described below
commit 984dfc89fda7276c240ddc4d5bc5828c1b3fd084
Author: Shawn Yang <[email protected]>
AuthorDate: Thu Dec 4 16:16:51 2025 +0800
feat(python): parallel python wheel build (#2989)
## Why?
## What does this PR do?
## Related issues
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
---
.github/workflows/build-containerized-pr.yml | 5 +-
.github/workflows/build-containerized-release.yml | 25 +++---
.github/workflows/build-native-pr.yml | 2 +-
.gitignore | 4 +-
ci/build_linux_wheels.py | 98 ++++++++---------------
python/pyfory/tests/test_buffer.py | 1 +
6 files changed, 51 insertions(+), 84 deletions(-)
diff --git a/.github/workflows/build-containerized-pr.yml
b/.github/workflows/build-containerized-pr.yml
index 3f1240f1d..837a50811 100644
--- a/.github/workflows/build-containerized-pr.yml
+++ b/.github/workflows/build-containerized-pr.yml
@@ -29,12 +29,13 @@ jobs:
strategy:
matrix:
os: [ubuntu-latest, ubuntu-24.04-arm]
+ python: [cp313-cp313]
steps:
- uses: actions/checkout@v5
- name: Build and test wheels
- run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }}
+ run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} --python ${{
matrix.python }}
- name: Upload wheels as artifacts
uses: actions/upload-artifact@v4
with:
- name: pyfory-wheels-containerized-${{ matrix.os }}
+ name: pyfory-wheels-${{ matrix.os }}-${{ matrix.python }}
path: dist/*.whl
diff --git a/.github/workflows/build-containerized-release.yml
b/.github/workflows/build-containerized-release.yml
index 7f9a78b23..513a1cd99 100644
--- a/.github/workflows/build-containerized-release.yml
+++ b/.github/workflows/build-containerized-release.yml
@@ -26,29 +26,22 @@ jobs:
strategy:
matrix:
os: [ubuntu-latest, ubuntu-24.04-arm]
+ python:
+ - cp38-cp38
+ - cp39-cp39
+ - cp310-cp310
+ - cp311-cp311
+ - cp312-cp312
+ - cp313-cp313
steps:
- uses: actions/checkout@v5
- name: Bump version
# Pass the tag name from the push (e.g. "v0.12.1b1"); deploy.sh will
strip leading "v".
run: ./ci/deploy.sh bump_py_version "${{ github.ref_name }}"
- - name: Set up Python 3.8
- uses: actions/setup-python@v5
- with:
- python-version: 3.8
- cache: 'pip'
- - name: Cache Bazel binary
- uses: actions/cache@v4
- with:
- path: |
- ~/bin/bazel
- ~/.local/bin/bazel
- key: bazel-binary-${{ runner.os }}-${{ runner.arch }}-${{
hashFiles('.bazelversion') }}
- - name: Install bazel
- run: ./ci/run_ci.sh install_bazel
- name: Build and test wheels
- run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} --release
+ run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} --python ${{
matrix.python }} --release
- name: Upload wheels as artifacts
uses: actions/upload-artifact@v4
with:
- name: pyfory-wheels-${{ matrix.os }}-${{ runner.arch }}-${{
github.ref_name }}
+ name: pyfory-wheels-${{ matrix.os }}-${{ matrix.python }}-${{
github.ref_name }}
path: dist/*.whl
diff --git a/.github/workflows/build-native-pr.yml
b/.github/workflows/build-native-pr.yml
index e7f4131a9..044c8f077 100644
--- a/.github/workflows/build-native-pr.yml
+++ b/.github/workflows/build-native-pr.yml
@@ -29,7 +29,7 @@ jobs:
strategy:
matrix:
os: [macos-latest, windows-latest]
- python-version: ['3.8', '3.13']
+ python-version: ['3.13']
steps:
- uses: actions/checkout@v5
- uses: actions/setup-python@v5
diff --git a/.gitignore b/.gitignore
index 558badab6..6e70e0eb9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,8 @@
**/*.dylib
**/*.pyd
bazel-*
+.bazel_user_root/
+**/.bazel_user_root/
.whl
python/.cache
python/pyfory/__pycache__/
@@ -80,4 +82,4 @@ dist/
**/.pytest_cache
.venv
**/.idea
-examples/cpp/cmake_example/build
\ No newline at end of file
+examples/cpp/cmake_example/build
diff --git a/ci/build_linux_wheels.py b/ci/build_linux_wheels.py
index d1ba2cb5d..54fdc2ed3 100755
--- a/ci/build_linux_wheels.py
+++ b/ci/build_linux_wheels.py
@@ -18,14 +18,18 @@
# under the License.
"""
-Host-side wrapper: workflow provides only --arch.
-Images are defined as regular Python lists (no env vars).
+Host-side wrapper for building Python wheels in manylinux containers.
+
+Usage:
+ ./build_linux_wheels.py --arch X86 --python cp38-cp38
+ ./build_linux_wheels.py --arch AARCH64 --python cp313-cp313 --release
Environment:
- GITHUB_WORKSPACE (optional; defaults to cwd)
"""
from __future__ import annotations
+
import argparse
import os
import shlex
@@ -33,27 +37,15 @@ import subprocess
import sys
from typing import List
-# Define Python version sets directly in the Python script
-RELEASE_PYTHON_VERSIONS = (
- "cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313"
-)
-DEFAULT_PYTHON_VERSIONS = "cp38-cp38 cp313-cp313"
-
# Path to the container build script
CONTAINER_SCRIPT_PATH = "ci/tasks/python_container_build_script.sh"
DEFAULT_X86_IMAGES = [
"quay.io/pypa/manylinux2014_x86_64:latest",
- # "quay.io/pypa/manylinux_2_28_x86_64:latest",
- # bazel binaries do not work with musl
- # "quay.io/pypa/musllinux_1_2_x86_64:latest",
]
DEFAULT_AARCH64_IMAGES = [
"quay.io/pypa/manylinux2014_aarch64:latest",
- # "quay.io/pypa/manylinux_2_28_aarch64:latest",
- # bazel binaries do not work with musl
- # "quay.io/pypa/musllinux_1_2_aarch64:latest",
]
ARCH_ALIASES = {
@@ -73,10 +65,11 @@ def parse_args():
"--arch", required=True, help="Architecture (e.g. X86, X64, AARCH64)"
)
p.add_argument(
- "--release", action="store_true", help="Run full test suite for
release"
+ "--python", required=True, help="Python version (e.g. cp38-cp38,
cp313-cp313)"
)
+ p.add_argument("--release", action="store_true", help="Run in release
mode")
p.add_argument(
- "--dry-run", action="store_true", help="Print docker commands without
running"
+ "--dry-run", action="store_true", help="Print docker command without
running"
)
return p.parse_args()
@@ -86,21 +79,19 @@ def normalize_arch(raw: str) -> str:
return ARCH_ALIASES.get(key, raw.strip().lower())
-def collect_images_for_arch(arch_normalized: str) -> List[str]:
+def get_image_for_arch(arch_normalized: str) -> str:
if arch_normalized == "x86":
- imgs = DEFAULT_X86_IMAGES # dedupe preserving order
+ return DEFAULT_X86_IMAGES[0]
elif arch_normalized == "arm64":
- imgs = DEFAULT_AARCH64_IMAGES
+ return DEFAULT_AARCH64_IMAGES[0]
else:
raise SystemExit(f"Unsupported arch: {arch_normalized!r}")
- return imgs
-def build_docker_cmd(workspace: str, image: str, release: bool = False) ->
List[str]:
+def build_docker_cmd(
+ workspace: str, image: str, python_version: str, release: bool = False
+) -> List[str]:
workspace = os.path.abspath(workspace)
- python_versions = RELEASE_PYTHON_VERSIONS if release else
DEFAULT_PYTHON_VERSIONS
-
- # Get GitHub reference name from environment
github_ref_name = os.environ.get("GITHUB_REF_NAME", "")
cmd = [
@@ -109,16 +100,15 @@ def build_docker_cmd(workspace: str, image: str, release:
bool = False) -> List[
"-i",
"--rm",
"-v",
- f"{workspace}:/work", # (v)olume
+ f"{workspace}:/work",
"-w",
- "/work", # (w)orking directory
+ "/work",
"-e",
- f"PYTHON_VERSIONS={python_versions}", # (e)nvironment variables
+ f"PYTHON_VERSIONS={python_version}",
"-e",
f"RELEASE_BUILD={'1' if release else '0'}",
]
- # Pass GitHub reference name if available
if github_ref_name:
cmd.extend(["-e", f"GITHUB_REF_NAME={github_ref_name}"])
@@ -126,52 +116,32 @@ def build_docker_cmd(workspace: str, image: str, release:
bool = False) -> List[
return cmd
-def run_for_images(
- images: List[str], workspace: str, dry_run: bool, release: bool = False
-) -> int:
- rc_overall = 0
- for image in images:
- docker_cmd = build_docker_cmd(workspace, image, release=release)
- printable = " ".join(shlex.quote(c) for c in docker_cmd)
- print(f"+ {printable}")
- if dry_run:
- continue
- try:
- completed = subprocess.run(docker_cmd)
- if completed.returncode != 0:
- print(
- f"Container {image} exited with {completed.returncode}",
- file=sys.stderr,
- )
- rc_overall = completed.returncode if rc_overall == 0 else
rc_overall
- else:
- print(f"Container {image} completed successfully.")
- except KeyboardInterrupt:
- print("Interrupted by user", file=sys.stderr)
- return 130
- except FileNotFoundError as e:
- print(f"Error running docker: {e}", file=sys.stderr)
- return 2
- return rc_overall
-
-
def main() -> int:
args = parse_args()
arch = normalize_arch(args.arch)
- images = collect_images_for_arch(arch)
- if not images:
- print(f"No images configured for arch {arch}", file=sys.stderr)
- return 2
+ image = get_image_for_arch(arch)
workspace = os.environ.get("GITHUB_WORKSPACE", os.getcwd())
- # Check if the container script exists
script_path = os.path.join(workspace, CONTAINER_SCRIPT_PATH)
if not os.path.exists(script_path):
print(f"Container script not found at {script_path}", file=sys.stderr)
return 2
- print(f"Selected images for arch {args.arch}: {images}")
- return run_for_images(images, workspace, args.dry_run,
release=args.release)
+ docker_cmd = build_docker_cmd(workspace, image, args.python,
release=args.release)
+ printable = " ".join(shlex.quote(c) for c in docker_cmd)
+ print(f"+ {printable}")
+
+ if args.dry_run:
+ return 0
+
+ try:
+ completed = subprocess.run(docker_cmd)
+ if completed.returncode != 0:
+ print(f"Container exited with {completed.returncode}",
file=sys.stderr)
+ return completed.returncode
+ except FileNotFoundError as e:
+ print(f"Error running docker: {e}", file=sys.stderr)
+ return 2
if __name__ == "__main__":
diff --git a/python/pyfory/tests/test_buffer.py
b/python/pyfory/tests/test_buffer.py
index cefd6abf5..f034c2c53 100644
--- a/python/pyfory/tests/test_buffer.py
+++ b/python/pyfory/tests/test_buffer.py
@@ -138,6 +138,7 @@ def check_varint32(buf: Buffer, value: int):
@require_pyarrow
def test_buffer_protocol():
+ # test buffer protocol compatibility with pyarrow
buffer = Buffer.allocate(32)
binary = b"b" * 100
buffer.write_bytes_and_size(binary)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
