This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch tirx/standalone-package-cleanup in repository https://gitbox.apache.org/repos/asf/tvm.git
commit ded2e085a6b161365f5d8338a0f2faa0a0866e83 Author: Tianqi Chen <[email protected]> AuthorDate: Thu Oct 15 02:10:47 2026 +0000 [FIX][CI] Build both native partitions with the pinned FFI dependency --- ci/jenkins/data.py | 4 +++- tests/scripts/task_build.py | 45 ++++++++++++++++++++++++++++------- tests/scripts/task_cpp_unittest.sh | 6 +++++ tests/scripts/task_python_unittest.sh | 4 ++-- 4 files changed, 48 insertions(+), 11 deletions(-) diff --git a/ci/jenkins/data.py b/ci/jenkins/data.py index 3f6f16959b..f4e4b632cc 100644 --- a/ci/jenkins/data.py +++ b/ci/jenkins/data.py @@ -29,6 +29,7 @@ data-driven artifact lists. """ import sys +from glob import glob files_to_stash = { # Executables and build files needed to run c++ tests @@ -45,7 +46,8 @@ files_to_stash = { "build/lib/libtvm_compiler.so", "build/lib/libtvm_tirx.so", "build/lib/libtvm_runtime.so", - "build/lib/libtvm_ffi.so", + "python/tvm_ffi", + *glob("python/apache_tvm_ffi-*.dist-info"), "build/lib/libtvm_runtime_cuda.so", "build/lib/libtvm_runtime_vulkan.so", "build/lib/libtvm_runtime_opencl.so", diff --git a/tests/scripts/task_build.py b/tests/scripts/task_build.py index c6c30c9bbd..47fe67efd6 100755 --- a/tests/scripts/task_build.py +++ b/tests/scripts/task_build.py @@ -20,7 +20,9 @@ import argparse import logging import multiprocessing import os +import shlex import shutil +import subprocess import sys from pathlib import Path @@ -36,19 +38,34 @@ if __name__ == "__main__": parser.add_argument("--sccache-bucket", required=False, help="sccache bucket name") parser.add_argument("--sccache-region", required=False, help="sccache region") parser.add_argument("--build-dir", default="build", help="build folder") - parser.add_argument("--cmake-target", help="optional build target") + parser.add_argument( + "--cmake-target", action="append", default=[], help="build target (repeatable)" + ) + parser.add_argument( + "--ffi-only", action="store_true", help="prepare the source-pinned FFI package" + ) parser.add_argument("--debug", required=False, action="store_true", help="build in debug mode") args = parser.parse_args() sccache_exe = shutil.which("sccache") - if args.cmake_target in ["standalone_crt", "crttest"]: + if args.cmake_target and all( + target in {"standalone_crt", "crttest"} for target in args.cmake_target + ): logging.info("Skipping standalone_crt build") exit(0) use_sccache = sccache_exe is not None build_dir = Path(os.getcwd()) / args.build_dir build_dir = build_dir.relative_to(REPO_ROOT) - env = {} + env = { + "PYTHONPATH": os.pathsep.join( + [ + str(REPO_ROOT / "tirx" / "python"), + str(REPO_ROOT / "python"), + os.environ.get("PYTHONPATH", ""), + ] + ) + } if use_sccache: if args.sccache_bucket and "AWS_ACCESS_KEY_ID" in os.environ: @@ -72,6 +89,22 @@ if __name__ == "__main__": sh = Sh(env) + # Build from this checkout's pinned submodule once, then transfer the exact + # Python/native package together with the TVM libraries to test workers. + ffi_commit = subprocess.check_output( + ["git", "-C", str(REPO_ROOT / "3rdparty" / "tvm-ffi"), "rev-parse", "HEAD"], text=True + ).strip() + ffi_identity = f"{ffi_commit} {sys.implementation.cache_tag}" + ffi_marker = REPO_ROOT / "python" / "tvm_ffi" / ".source-commit" + if not ffi_marker.is_file() or ffi_marker.read_text().strip() != ffi_identity: + sh.run( + f"uv pip install --python {shlex.quote(sys.executable)} " + "-v --no-deps --reinstall --target=python ./3rdparty/tvm-ffi/" + ) + ffi_marker.write_text(ffi_identity + "\n") + if args.ffi_only: + sys.exit(0) + if use_sccache: sh.run("sccache --start-server", check=False) logging.info("===== sccache stats =====") @@ -90,12 +123,8 @@ if __name__ == "__main__": else: sh.run("cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo ..", cwd=build_dir) - target = "" - if args.cmake_target: - target = args.cmake_target - verbose = os.environ.get("VERBOSE", "true").lower() in {"1", "true", "yes"} - ninja_args = [target, f"-j{num_cpus}"] + ninja_args = [*args.cmake_target, f"-j{num_cpus}"] if verbose: ninja_args.append("-v") diff --git a/tests/scripts/task_cpp_unittest.sh b/tests/scripts/task_cpp_unittest.sh index 424642673b..efe6417733 100755 --- a/tests/scripts/task_cpp_unittest.sh +++ b/tests/scripts/task_cpp_unittest.sh @@ -32,6 +32,12 @@ fi export TVM_BIND_THREADS=0 export OMP_NUM_THREADS=1 +# Test workers receive the exact FFI package in their own checkout. Native +# binaries must find that copy even when their build workspace had another path. +ffi_lib_dir="$(pwd)/python/tvm_ffi/lib" +export LD_LIBRARY_PATH="${ffi_lib_dir}:${LD_LIBRARY_PATH:-}" +export DYLD_LIBRARY_PATH="${ffi_lib_dir}:${DYLD_LIBRARY_PATH:-}" + pushd "${BUILD_DIR}" # run cpp test executable ./cpptest diff --git a/tests/scripts/task_python_unittest.sh b/tests/scripts/task_python_unittest.sh index c6c578cc5b..85b5d692f0 100755 --- a/tests/scripts/task_python_unittest.sh +++ b/tests/scripts/task_python_unittest.sh @@ -21,8 +21,8 @@ set -euxo pipefail export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" export PYTEST_ADDOPTS="${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" -# setup tvm-ffi into python folder -uv pip install -v --target=python ./3rdparty/tvm-ffi/ +# Reuse the exact source-pinned FFI package transferred by the native build. +python3 tests/scripts/task_build.py --ffi-only # LLVMModule JIT execution. Installed here as well as in the CI image because a docker/ # change does not reach the test containers in the same run (they use the pinned tag).
