This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch jenkins-wheel-test-artifacts in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 61ec4c3bf1a3faea2a893d49ca90738147aed3fa Author: Tianqi Chen <[email protected]> AuthorDate: Thu Oct 15 18:15:51 2026 +0000 [CI] Build and consume source wheel artifacts across Jenkins platforms --- CMakeLists.txt | 9 ++ ci/README.md | 10 ++ ci/jenkins/generated/arm_jenkinsfile.groovy | 31 ++-- ci/jenkins/generated/common.groovy | 23 ++- ci/jenkins/generated/cpu_jenkinsfile.groovy | 35 ++--- ci/jenkins/generated/gpu_jenkinsfile.groovy | 23 ++- ci/jenkins/generated/wasm_jenkinsfile.groovy | 32 +--- docker/bash.sh | 8 + docs/conf.py | 5 +- .../linux-x86_64/src/main/assembly/assembly.xml | 7 +- .../osx-x86_64/src/main/assembly/assembly.xml | 7 +- jvm/core/pom.xml | 2 +- jvm/native/linux-x86_64/pom.xml | 6 +- jvm/native/osx-x86_64/pom.xml | 6 +- jvm/pom.xml | 4 + pyproject.toml | 3 + python/tvm/__init__.py | 32 ++-- python/tvm/config.py | 97 ++++++++++++ python/tvm/rpc/minrpc.py | 2 + tests/python/runtime/rpc/test_minrpc_source.py | 51 +++++++ tests/python/runtime/test_config.py | 74 +++++++++ tests/scripts/ci.py | 85 ++++------- tests/scripts/task_build.py | 143 ----------------- .../{task_web_wasm.sh => task_build_arm.sh} | 27 ++-- ...{task_config_build_arm.sh => task_build_cpu.sh} | 28 ++-- ...task_web_wasm.sh => task_build_dependencies.sh} | 24 +-- ...config_build_gpu_other.sh => task_build_gpu.sh} | 32 ++-- ...config_build_cpu.sh => task_build_gpu_other.sh} | 34 +++-- ...onfig_build_gpu_other.sh => task_build_wasm.sh} | 23 ++- tests/scripts/task_build_wheels.sh | 47 ++++++ tests/scripts/task_config_build_gpu.sh | 43 ------ ...d_wasm.sh => task_install_docs_dependencies.sh} | 24 +-- tests/scripts/task_java_unittest.sh | 8 +- tests/scripts/task_jvm_build.sh | 22 ++- tests/scripts/task_python_docs.sh | 21 +-- tests/scripts/task_python_install_wheels.sh | 32 ++++ tests/scripts/task_python_unittest.sh | 26 ++-- tests/scripts/task_python_unittest_gpuonly.sh | 5 +- tests/scripts/task_web_wasm.sh | 10 +- tirx/pyproject.toml | 3 + tirx/python/tvm_tirx/__init__.py | 169 ++++++++++++--------- tirx/python/tvm_tirx/config.py | 99 ++++++++++++ tirx/tests/python/runtime/test_tirx_config.py | 74 +++++++++ .../tests/scripts/task_build_arm.sh | 20 ++- .../tests/scripts/task_build_cpu.sh | 26 ++-- .../tests/scripts/task_build_dependencies.sh | 27 ++-- .../tests/scripts/task_build_gpu.sh | 31 ++-- .../tests/scripts/task_build_wasm.sh | 21 ++- tirx/tests/scripts/task_build_wheels.sh | 55 +++++++ 49 files changed, 1012 insertions(+), 614 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4bd18b92a0..c24451877f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -294,6 +294,15 @@ install(TARGETS tvm_compiler tvm_runtime_extra ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR}) +foreach(_target fpA_intB_gemm flash_attn) + if(TARGET ${_target}) + tvm_configure_target_library(${_target}) + install(TARGETS ${_target} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR}) + endif() +endforeach() install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} FILES_MATCHING PATTERN "*.h") # Parent minRPC compilation retains its own source and private header closure. install(DIRECTORY src/runtime/rpc/ DESTINATION src/runtime/rpc FILES_MATCHING PATTERN "*.h" PATTERN "*.cc") diff --git a/ci/README.md b/ci/README.md index e617d7af91..a9e65f3622 100644 --- a/ci/README.md +++ b/ci/README.md @@ -93,3 +93,13 @@ Other jobs run in GitHub Actions for Windows and MacOS jobs. The native Groovy entry points in `ci/jenkins/generated/` load shared helpers from `common.groovy`. Edit these sources directly; no generation step is needed. See comments in [common.groovy](jenkins/generated/common.groovy) for shared utility options and trust constraints. + +The native Jenkins pipelines call `tests/scripts/task_build_<platform>.sh`. +Each root script invokes its counterpart under `tirx/tests/scripts/` to build +pinned-source FFI and TIRx wheels, then builds TVM against that installed core. All three wheel artifacts share +the checkout-root `dist/` directory. Platform options use direct CMake `-D` flags. +C++ tests run in the native build that produced each wheel. Consumers install the +three transferred wheels offline without dependencies into `.local/python` and +use its absolute path as `PYTHONPATH`; `TVM_HOME` retains checkout resources. +Groovy sets the cache, parallelism and C/C++ compiler-launcher environment values; +`docker/bash.sh` forwards them to the build scripts. diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy index 1429442a03..cdfdac1d79 100644 --- a/ci/jenkins/generated/arm_jenkinsfile.groovy +++ b/ci/jenkins/generated/arm_jenkinsfile.groovy @@ -27,30 +27,19 @@ try { ci = evaluate(helper_source) ci.start() +docker_run += ' --env TVM_TEST_USE_WHEEL=1' + stage('Build') { ci.with_node(instance: 'ARM-GRAVITON3-SPOT', retry_instance: 'ARM-GRAVITON3', name: 'BUILD: arm', folder: 'tvm/build-arm', variables: ['PLATFORM=arm']) { ci.docker_init(images.ci_arm) - sh( - script: "${docker_run} ${images.ci_arm} ./tests/scripts/task_config_build_arm.sh build", - label: 'Create ARM CMake config', - ) - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${images.ci_arm} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir build", - label: 'Run CMake build', - ) - sh ( - script: """ - set -eux - ${docker_run} ${images.ci_arm} python3 ./tests/scripts/task_build.py \ - --sccache-bucket tvm-sccache-prod \ - --sccache-region us-west-2 \ - --cmake-target cpptest \ - --cmake-target tirx_cpptest \ - --build-dir build - """, - label: 'Make C++ tests', - ) - ci.upload_artifacts('arm', '--artifact tvm_lib --artifact cpptest') + withEnv(ci.build_environment()) { + sh( + script: "${docker_run} ${images.ci_arm} ./tests/scripts/task_build_arm.sh build 1", + label: 'Build arm wheels and C++ tests', + ) + } + ci.upload_artifacts('arm', '--path "dist/*.whl"') + } } diff --git a/ci/jenkins/generated/common.groovy b/ci/jenkins/generated/common.groovy index 1e9ce344be..e6be4ed05f 100644 --- a/ci/jenkins/generated/common.groovy +++ b/ci/jenkins/generated/common.groovy @@ -22,8 +22,19 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils image_platforms = [ci_arm: 'ARM', ci_cpu: 'CPU', ci_gpu: 'GPU', ci_wasm: 'CPU'] // Resolve Hub inputs through the caller account ECR cache; false keeps public names. enable_docker_hub_cache = true +// Preserve local caching when the container has no explicit AWS credentials. +if (env.AWS_ACCESS_KEY_ID) { + env.SCCACHE_BUCKET = env.SCCACHE_BUCKET ?: 'tvm-sccache-prod' +} +env.SCCACHE_REGION = env.SCCACHE_REGION ?: 'us-west-2' +// Images also expose sccache as cc/c++; keep the compiler separate from its launcher. +env.CC = env.CC ?: 'gcc' +env.CXX = env.CXX ?: 'g++' +env.CMAKE_C_COMPILER_LAUNCHER = env.CMAKE_C_COMPILER_LAUNCHER ?: 'sccache' +env.CMAKE_CXX_COMPILER_LAUNCHER = env.CMAKE_CXX_COMPILER_LAUNCHER ?: 'sccache' max_time = 180 -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TEST_STEP_NAME' + + ' --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_SESSION_TOKEN' docker_build = 'docker/build.sh' s3_bucket = 'tvm-jenkins-artifacts-prod' jenkins_scripts_root = 'ci/jenkins/scripts' @@ -388,4 +399,14 @@ def start() { } } +// Preserve caller limits, otherwise share the node among its executors. +def build_environment() { + def parallel_limit = env.CMAKE_BUILD_PARALLEL_LEVEL ?: sh( + script: 'n=$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} )); if [ "$n" -lt 1 ]; then n=1; fi; echo "$n"', + returnStdout: true, + label: 'Determine build parallelism', + ).trim() + return ["CMAKE_BUILD_PARALLEL_LEVEL=${parallel_limit}"] +} + return this diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy index 44aceb116d..91a8510a63 100644 --- a/ci/jenkins/generated/cpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy @@ -27,31 +27,20 @@ try { ci = evaluate(helper_source) ci.start() +docker_run += ' --env TVM_TEST_USE_WHEEL=1' + stage('Build') { ci.with_node(instance: 'CPU-SPOT', retry_instance: 'CPU', name: 'BUILD: CPU', folder: 'tvm/build-cpu', variables: ['PLATFORM=cpu']) { ci.docker_init(images.ci_cpu) - sh( - script: "${docker_run} ${images.ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", - label: 'Create CPU CMake config', - ) - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${images.ci_cpu} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir build", - label: 'Run CMake build', - ) - sh ( - script: """ - set -eux - ${docker_run} ${images.ci_cpu} python3 ./tests/scripts/task_build.py \ - --sccache-bucket tvm-sccache-prod \ - --sccache-region us-west-2 \ - --cmake-target cpptest \ - --cmake-target tirx_cpptest \ - --build-dir build - """, - label: 'Make C++ tests', - ) - ci.upload_artifacts('cpu', '--artifact tvm_lib --artifact cpptest') + withEnv(ci.build_environment()) { + sh( + script: "${docker_run} ${images.ci_cpu} ./tests/scripts/task_build_cpu.sh build 1", + label: 'Build cpu wheels and C++ tests', + ) + } + ci.upload_artifacts('cpu', '--path "dist/*.whl"') + } } @@ -62,10 +51,6 @@ stage('Test') { variables: ['PLATFORM=cpu', 'TEST_STEP_NAME=unittest: CPU']) { ci.docker_init(images.ci_cpu) ci.download_artifacts('cpu') - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${images.ci_cpu} ./tests/scripts/task_cpp_unittest.sh", - label: 'Run C++ tests', - ) sh ( script: "${docker_run} ${images.ci_cpu} ./tests/scripts/task_python_unittest.sh", label: 'Run Python unit tests', diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy index ac46fa4de1..391dd9384c 100644 --- a/ci/jenkins/generated/gpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy @@ -29,21 +29,28 @@ try { ci = evaluate(helper_source) ci.start() +docker_run += ' --env TVM_TEST_USE_WHEEL=1' + stage('Build') { ci.with_node(instance: 'CPU-SPOT', retry_instance: 'CPU', name: 'BUILD: GPU', folder: 'tvm/build-gpu', variables: ['PLATFORM=gpu']) { ci.docker_init(images.ci_gpu) - sh "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${images.ci_gpu} --no-gpu ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir build", - label: 'Run CMake build', - ) - ci.upload_artifacts('gpu', '--artifact tvm_lib --artifact tvm_lib_gpu_extra') + withEnv(ci.build_environment()) { + sh( + script: "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_build_gpu.sh build 0", + label: 'Build gpu', + ) + } + ci.upload_artifacts('gpu', '--path "dist/*.whl"') // Check the compiler-only GPU configuration after publishing the test build. sh 'rm -rf build' - sh "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build" - sh "${docker_run} --env CI_NUM_EXECUTORS ${images.ci_gpu} --no-gpu ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --cmake-target tvm_runtime --build-dir build" + withEnv(ci.build_environment()) { + sh( + script: "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_build_gpu_other.sh build 0", + label: 'Build gpu_other', + ) + } } } diff --git a/ci/jenkins/generated/wasm_jenkinsfile.groovy b/ci/jenkins/generated/wasm_jenkinsfile.groovy index 593da51094..681c92184d 100644 --- a/ci/jenkins/generated/wasm_jenkinsfile.groovy +++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy @@ -27,34 +27,18 @@ try { ci = evaluate(helper_source) ci.start() +docker_run += ' --env TVM_TEST_USE_WHEEL=1' + stage('Build') { ci.with_node(instance: 'CPU-SPOT', retry_instance: 'CPU', name: 'BUILD: WASM', folder: 'tvm/build-wasm', variables: ['PLATFORM=wasm']) { ci.docker_init(images.ci_wasm) - sh( - script: "${docker_run} ${images.ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", - label: 'Create WASM CMake config', - ) - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${images.ci_wasm} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir build", - label: 'Run CMake build', - ) - sh ( - script: """ - set -eux - ${docker_run} ${images.ci_wasm} python3 ./tests/scripts/task_build.py \ - --sccache-bucket tvm-sccache-prod \ - --sccache-region us-west-2 \ - --cmake-target cpptest \ - --cmake-target tirx_cpptest \ - --build-dir build - """, - label: 'Make C++ tests', - ) - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${images.ci_wasm} ./tests/scripts/task_cpp_unittest.sh", - label: 'Run C++ tests', - ) + withEnv(ci.build_environment()) { + sh( + script: "${docker_run} ${images.ci_wasm} ./tests/scripts/task_build_wasm.sh build 1", + label: 'Build wasm wheels and C++ tests', + ) + } sh( script: "${docker_run} ${images.ci_wasm} ./tests/scripts/task_web_wasm.sh", label: 'Run WASM lint and tests', diff --git a/docker/bash.sh b/docker/bash.sh index fe0b1eeb2b..639c002761 100755 --- a/docker/bash.sh +++ b/docker/bash.sh @@ -236,6 +236,14 @@ function validate_mount() { break_joined_flag='if (( ${#1} == 2 )); then shift; else set -- -"${1#-i}" "${@:2}"; fi' DOCKER_ENV=( ) +# Forward build settings selected by the caller without overriding explicit values. +BUILD_ENV_VARS=(CC CXX SCCACHE_BUCKET SCCACHE_REGION CMAKE_BUILD_PARALLEL_LEVEL + CMAKE_C_COMPILER_LAUNCHER CMAKE_CXX_COMPILER_LAUNCHER) +for build_var in "${BUILD_ENV_VARS[@]}"; do + if [ "${!build_var+x}" = "x" ]; then + DOCKER_ENV+=( --env "$build_var=${!build_var}" ) + fi +done DOCKER_FLAGS=( ) while (( $# )); do diff --git a/docs/conf.py b/docs/conf.py index 361356a95e..0031d0b501 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -50,8 +50,9 @@ if curr_path.name == "_staging": else: tvm_path = Path(os.pardir) -sys.path.insert(0, str(tvm_path.resolve() / "python")) -sys.path.insert(0, str(tvm_path.resolve() / "tirx" / "python")) +if os.environ.get("TVM_DOCS_USE_WHEEL") != "1": + sys.path.insert(0, str(tvm_path.resolve() / "python")) + sys.path.insert(0, str(tvm_path.resolve() / "tirx" / "python")) sys.path.insert(0, str(tvm_path.resolve() / "docs")) # -- General configuration ------------------------------------------------ diff --git a/jvm/assembly/linux-x86_64/src/main/assembly/assembly.xml b/jvm/assembly/linux-x86_64/src/main/assembly/assembly.xml index 7345c496e9..ea7396f8dc 100644 --- a/jvm/assembly/linux-x86_64/src/main/assembly/assembly.xml +++ b/jvm/assembly/linux-x86_64/src/main/assembly/assembly.xml @@ -26,7 +26,12 @@ under the License. <includeBaseDirectory>false</includeBaseDirectory> <files> <file> - <source>../../../build/lib/libtvm_runtime.so</source> + <source>${tvm.lib.dir}/libtvm_runtime.so</source> + <outputDirectory>lib/native</outputDirectory> + <fileMode>0644</fileMode> + </file> + <file> + <source>${tvm.ffi.lib.dir}/libtvm_ffi.so</source> <outputDirectory>lib/native</outputDirectory> <fileMode>0644</fileMode> </file> diff --git a/jvm/assembly/osx-x86_64/src/main/assembly/assembly.xml b/jvm/assembly/osx-x86_64/src/main/assembly/assembly.xml index 739852e474..70f5754983 100644 --- a/jvm/assembly/osx-x86_64/src/main/assembly/assembly.xml +++ b/jvm/assembly/osx-x86_64/src/main/assembly/assembly.xml @@ -26,7 +26,12 @@ under the License. <includeBaseDirectory>false</includeBaseDirectory> <files> <file> - <source>../../../build/lib/libtvm_runtime.dylib</source> + <source>${tvm.lib.dir}/libtvm_runtime.dylib</source> + <outputDirectory>lib/native</outputDirectory> + <fileMode>0644</fileMode> + </file> + <file> + <source>${tvm.ffi.lib.dir}/libtvm_ffi.dylib</source> <outputDirectory>lib/native</outputDirectory> <fileMode>0644</fileMode> </file> diff --git a/jvm/core/pom.xml b/jvm/core/pom.xml index eeb55f5cfd..26ba6dcd78 100644 --- a/jvm/core/pom.xml +++ b/jvm/core/pom.xml @@ -104,7 +104,7 @@ under the License. <threadCount>1</threadCount> <argLine> -Djava.library.path=${project.parent.basedir}/native/${platform}/target - -Dlibtvm.so.path=${project.parent.basedir}/../build/lib/${libtvm.so.filename} + -Dlibtvm.so.path=${tvm.lib.dir}/${libtvm.so.filename} -Dlog4j.configuration=file://${project.parent.basedir}/conf/log4j.properties </argLine> </configuration> diff --git a/jvm/native/linux-x86_64/pom.xml b/jvm/native/linux-x86_64/pom.xml index 31e120bc58..dcda4c6ce1 100644 --- a/jvm/native/linux-x86_64/pom.xml +++ b/jvm/native/linux-x86_64/pom.xml @@ -117,8 +117,8 @@ under the License. <compilerStartOption>-std=c++17</compilerStartOption> </compilerStartOptions> <compilerEndOptions> - <compilerEndOption>-I../../../include</compilerEndOption> - <compilerEndOption>-I../../../3rdparty/tvm-ffi/include</compilerEndOption> + <compilerEndOption>-I${tvm.include.dir}</compilerEndOption> + <compilerEndOption>-I${tvm.ffi.include.dir}</compilerEndOption> <compilerEndOption>-I${JAVA_HOME}/include</compilerEndOption> <compilerEndOption>-I${JAVA_HOME}/include/linux</compilerEndOption> <compilerEndOption>${cflags}</compilerEndOption> @@ -127,7 +127,7 @@ under the License. <linkerStartOption>-shared</linkerStartOption> </linkerStartOptions> <linkerEndOptions> - <linkerEndOption>-L${project.parent.basedir}/../../build/lib/</linkerEndOption> + <linkerEndOption>-L${tvm.lib.dir}</linkerEndOption> <linkerEndOption>-ltvm_runtime</linkerEndOption> <linkerEndOption>${ldflags}</linkerEndOption> </linkerEndOptions> diff --git a/jvm/native/osx-x86_64/pom.xml b/jvm/native/osx-x86_64/pom.xml index 4f9d70d60d..ccb79f5bbb 100644 --- a/jvm/native/osx-x86_64/pom.xml +++ b/jvm/native/osx-x86_64/pom.xml @@ -118,8 +118,8 @@ under the License. <compilerStartOption>-std=c++17</compilerStartOption> </compilerStartOptions> <compilerEndOptions> - <compilerEndOption>-I../../../include</compilerEndOption> - <compilerEndOption>-I../../../3rdparty/tvm-ffi/include</compilerEndOption> + <compilerEndOption>-I${tvm.include.dir}</compilerEndOption> + <compilerEndOption>-I${tvm.ffi.include.dir}</compilerEndOption> <compilerEndOption>-I${JAVA_HOME}/include</compilerEndOption> <compilerEndOption>-I${JAVA_HOME}/include/darwin</compilerEndOption> <compilerEndOption>${cflags}</compilerEndOption> @@ -134,7 +134,7 @@ under the License. <linkerMiddleOption>-Wl,-x</linkerMiddleOption> </linkerMiddleOptions> <linkerEndOptions> - <linkerEndOption>-L${project.parent.basedir}/../../build/lib/</linkerEndOption> + <linkerEndOption>-L${tvm.lib.dir}</linkerEndOption> <linkerEndOption>-ltvm_runtime</linkerEndOption> <linkerEndOption>${ldflags}</linkerEndOption> </linkerEndOptions> diff --git a/jvm/pom.xml b/jvm/pom.xml index e5efce0bf5..2a2f53cd73 100644 --- a/jvm/pom.xml +++ b/jvm/pom.xml @@ -44,6 +44,10 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <tvm.include.dir>${maven.multiModuleProjectDirectory}/../include</tvm.include.dir> + <tvm.ffi.include.dir>${maven.multiModuleProjectDirectory}/../3rdparty/tvm-ffi/include</tvm.ffi.include.dir> + <tvm.lib.dir>${maven.multiModuleProjectDirectory}/../build/lib</tvm.lib.dir> + <tvm.ffi.lib.dir>${tvm.lib.dir}</tvm.ffi.lib.dir> </properties> <packaging>pom</packaging> diff --git a/pyproject.toml b/pyproject.toml index cd3a9547bf..3ff5135849 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,9 @@ classifiers = [ ] dependencies = ["apache-tvm-tirx==0.1.0a0"] +[project.scripts] +tvm-config = "tvm.config:__main__" + [project.optional-dependencies] torch = ["torch"] cuda = ["cuda-bindings"] diff --git a/python/tvm/__init__.py b/python/tvm/__init__.py index 7320967ffb..8952ca68b3 100644 --- a/python/tvm/__init__.py +++ b/python/tvm/__init__.py @@ -18,13 +18,27 @@ # pylint: disable=redefined-builtin, wildcard-import """TVM compiler dialects and runtime extensions built on TIRx.""" -from .libinfo import __version__ -from .base import _RUNTIME_ONLY -from . import runtime -from . import rpc +import sys -if not _RUNTIME_ONLY: - from . import te - from . import s_tir - from .driver import compile - from . import relax +from typing import TYPE_CHECKING + + +def _is_config_mode() -> bool: + """Recognize the config CLI before loading Python or native TVM components.""" + if sys.argv[0].endswith("tvm-config"): + return True + argv = getattr(sys, "orig_argv", []) + return any(arg == "-m" and argv[i + 1 : i + 2] == ["tvm.config"] for i, arg in enumerate(argv)) + + +if TYPE_CHECKING or not _is_config_mode(): + from .libinfo import __version__ + from .base import _RUNTIME_ONLY + from . import runtime + from . import rpc + + if not _RUNTIME_ONLY: + from . import te + from . import s_tir + from .driver import compile + from . import relax diff --git a/python/tvm/config.py b/python/tvm/config.py new file mode 100644 index 0000000000..c76ef90006 --- /dev/null +++ b/python/tvm/config.py @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Lightweight compiler and linker configuration for the TVM runtime.""" + +import argparse +import os +import sys +from pathlib import Path + + +def _runtime_root() -> Path: + """Locate this package without importing its native code.""" + return Path(__file__).resolve().parent + + +def _find_include() -> str: + package = _runtime_root() + for path in (package / "include", package.parent.parent / "include"): + if (path / "tvm" / "runtime").is_dir(): + return str(path.resolve()) + raise RuntimeError("Cannot find TVM runtime headers") + + +def _library_paths() -> list[Path]: + package = _runtime_root() + source = package.parent.parent + paths = [package / "lib", source / "build" / "lib", source / "lib"] + if os.environ.get("TVM_LIBRARY_PATH"): + paths.insert(0, Path(os.environ["TVM_LIBRARY_PATH"])) + for name in ("LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH", "PATH"): + paths.extend(Path(value) for value in os.environ.get(name, "").split(os.pathsep) if value) + return paths + + +def _find_runtime() -> str: + filename = {"win32": "tvm_runtime_extra.dll", "darwin": "libtvm_runtime_extra.dylib"}.get( + sys.platform, "libtvm_runtime_extra.so" + ) + for directory in _library_paths(): + path = directory / filename + if path.is_file(): + return str(path.resolve()) + raise RuntimeError("Cannot find libtvm_runtime_extra") + + +def __main__() -> None: + """Print paths and flags using the same conventions as tvm_ffi.config.""" + parser = argparse.ArgumentParser(description="Get TVM runtime build configuration") + for name, help_text in ( + ("includedir", "Print include directory"), + ("libdir", "Print library directory"), + ("libfiles", "Fully qualified library filenames"), + ("libs", "Libraries to be linked"), + ("cflags", "Print C flags"), + ("cxxflags", "Print C++ flags"), + ("ldflags", "Print linker flags"), + ): + parser.add_argument(f"--{name}", action="store_true", help=help_text) + args = parser.parse_args() + if len(sys.argv) == 1: + parser.print_help() + return + if args.includedir: + print(_find_include()) + if args.libdir or args.libfiles or args.libs or args.ldflags: + library = Path(_find_runtime()) + link_file = library.with_suffix(".lib") if sys.platform == "win32" else library + if args.libdir: + print(library.parent) + if args.libfiles: + print(link_file) + if args.cxxflags: + print(f"-I{_find_include()} -std=c++20") + if args.cflags: + print(f"-I{_find_include()}") + if args.libs: + print(link_file if sys.platform == "win32" else "-ltvm_runtime_extra") + if args.ldflags and sys.platform != "win32": + print(f"-L{library.parent}") + + +if __name__ == "__main__": + __main__() diff --git a/python/tvm/rpc/minrpc.py b/python/tvm/rpc/minrpc.py index 5885f75005..2b5ba261a2 100644 --- a/python/tvm/rpc/minrpc.py +++ b/python/tvm/rpc/minrpc.py @@ -41,6 +41,8 @@ def find_minrpc_server_libpath(server="posix_popen_server"): curr_dir = os.path.dirname(os.path.realpath(os.path.expanduser(__file__))) source_dir = os.path.abspath(os.path.join(curr_dir, "..", "..", "..")) roots = [os.path.join(curr_dir, ".."), source_dir] + if os.environ.get("TVM_HOME"): + roots.append(os.environ["TVM_HOME"]) candidates = [] for root in roots: minrpc_dir = os.path.join(root, "src", "runtime", "rpc", "minrpc") diff --git a/tests/python/runtime/rpc/test_minrpc_source.py b/tests/python/runtime/rpc/test_minrpc_source.py new file mode 100644 index 0000000000..0512d1cfa5 --- /dev/null +++ b/tests/python/runtime/rpc/test_minrpc_source.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Find checkout-only minrpc sources while importing an installed TVM wheel.""" + +from pathlib import Path + +import pytest + +from tvm import libinfo +from tvm.rpc import minrpc + + [email protected]("layout", ["source", "symlink", "wheel"]) +def test_minrpc_source_location(tmp_path, monkeypatch, layout): + checkout = tmp_path / "checkout" + minrpc_dir = checkout / "src/runtime/rpc/minrpc" + server = minrpc_dir / "posix_popen_server/posix_popen_server.cc" + server.parent.mkdir(parents=True) + server.write_text("// source fixture\n") + # A wheel's package location is unrelated to the source checkout. + package = tmp_path / "site-packages/tvm" if layout == "wheel" else checkout / "python/tvm" + if layout == "symlink": + package.mkdir(parents=True) + alias = tmp_path / "python-alias" + alias.symlink_to(checkout / "python", target_is_directory=True) + package = alias / "tvm" + monkeypatch.setattr(libinfo, "__file__", str(package / "libinfo.py")) + monkeypatch.setattr(minrpc, "__file__", str(package / "rpc/minrpc.py")) + monkeypatch.delenv("TVM_HOME", raising=False) + if layout == "wheel": + monkeypatch.setenv("TVM_HOME", str(checkout)) + directory, path = minrpc.find_minrpc_server_libpath() + assert Path(directory) == minrpc_dir + assert Path(path) == server + server.unlink() + with pytest.raises(RuntimeError, match="Cannot find minserver"): + minrpc.find_minrpc_server_libpath() diff --git a/tests/python/runtime/test_config.py b/tests/python/runtime/test_config.py new file mode 100644 index 0000000000..c039e08bd2 --- /dev/null +++ b/tests/python/runtime/test_config.py @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""The config command must work without importing native runtime dependencies.""" + +import os +import subprocess +import sys +from pathlib import Path + +import tvm +from tvm import config + + +def test_config_without_native_imports(tmp_path): + # A startup hook rejects native loads and dependencies even when they are installed. + (tmp_path / "sitecustomize.py").write_text( + "import sys\n" + "class BlockNative:\n" + " def find_spec(self, fullname, path=None, target=None):\n" + " if fullname.split('.')[0] in ('tvm_ffi', 'torch', 'numpy') or " + "fullname in ('tvm.base', 'tvm.runtime', 'tvm.backend', 'tvm.driver', 'tvm_tirx.base'):\n" + " raise AssertionError(fullname)\n" + "sys.meta_path.insert(0, BlockNative())\n" + "def audit(event, args):\n" + " if event == 'ctypes.dlopen': raise AssertionError(args)\n" + "sys.addaudithook(audit)\n" + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + ( + str(tmp_path), + str(Path(tvm.__file__).parent.parent), + ) + ) + command = [sys.executable, "-m", "tvm.config"] + include = subprocess.check_output([*command, "--includedir"], env=env, text=True).strip() + assert (Path(include) / "tvm" / "runtime").is_dir() + library = subprocess.check_output([*command, "--libfiles"], env=env, text=True).strip() + assert Path(library).is_file() + flags = subprocess.check_output( + [*command, "--cflags", "--cxxflags", "--libs", "--ldflags"], env=env, text=True + ) + assert f"-I{include}" in flags + assert "-std=c++20" in flags + + +def test_runtime_library_path_fallback(tmp_path, monkeypatch): + library = tmp_path / Path(config._find_runtime()).name + library.touch() + monkeypatch.setattr(config, "_runtime_root", lambda: tmp_path / "tvm") + monkeypatch.delenv("LD_LIBRARY_PATH", raising=False) + monkeypatch.delenv("DYLD_LIBRARY_PATH", raising=False) + monkeypatch.setenv("PATH", str(tmp_path)) + assert config._find_runtime() == str(library) + + +if __name__ == "__main__": + import tvm_tirx.testing + + tvm_tirx.testing.main() diff --git a/tests/scripts/ci.py b/tests/scripts/ci.py index fa027c150a..a9c71e05c1 100755 --- a/tests/scripts/ci.py +++ b/tests/scripts/ci.py @@ -165,27 +165,26 @@ def docker( """ check_docker() - # As sccache is added to these images these can be uncommented - sccache_images = { - "ci_gpu", - "ci_cpu", - # "ci_wasm", - "ci_cortexm", - "ci_arm", - "ci_riscv", - "ci_adreno", - } - - if image in sccache_images and os.getenv("USE_SCCACHE", "1") == "1": - scripts = [ - "sccache --start-server", - ] + scripts - # Set the C/C++ compiler so CMake picks them up in the build - env["CC"] = "/opt/sccache/cc" - env["CXX"] = "/opt/sccache/c++" + env.setdefault("CC", os.getenv("CC", "gcc")) + env.setdefault("CXX", os.getenv("CXX", "g++")) + if os.getenv("USE_SCCACHE", "1") == "1": + env.setdefault( + "CMAKE_C_COMPILER_LAUNCHER", os.getenv("CMAKE_C_COMPILER_LAUNCHER", "sccache") + ) + env.setdefault( + "CMAKE_CXX_COMPILER_LAUNCHER", os.getenv("CMAKE_CXX_COMPILER_LAUNCHER", "sccache") + ) env["SCCACHE_CACHE_SIZE"] = os.getenv("SCCACHE_CACHE_SIZE", "50G") env["SCCACHE_SERVER_PORT"] = os.getenv("SCCACHE_SERVER_PORT", "4226") + # Pass the bound through Docker; CMake and scikit-build honor it directly. + env.setdefault( + "CMAKE_BUILD_PARALLEL_LEVEL", + os.environ.get( + "CMAKE_BUILD_PARALLEL_LEVEL", + str(max(NPROC // int(os.environ.get("CI_NUM_EXECUTORS", "1")), 1)), + ), + ) env["PLATFORM"] = name docker_bash = REPO_ROOT / "docker" / "bash.sh" @@ -258,42 +257,12 @@ def docs( # TODO: Change this to tlcpack/docs once that is uploaded image = "ci_cpu" if docker_image is None else docker_image build_dir = get_build_dir("cpu") - config_script = " && ".join( - [ - f"mkdir -p {build_dir}", - f"pushd {build_dir}", - "cp ../cmake/config.cmake .", - "popd", - ] - ) - - # These are taken from the ci-gpu image via pip freeze, consult that - # if there are any changes: https://github.com/apache/tvm/tree/main/docs#native - requirements = [ - "Sphinx==8.1.3", - "sphinx_autodoc_annotation~=1.0", - "sphinx-gallery==0.20.0", - "sphinx-book-theme==1.1.4", - "pydata-sphinx-theme==0.15.4", - "autodocsumm==0.2.14", - "image==1.5.33", - "matplotlib==3.10.8", - "commonmark==0.9.1", - "docutils==0.21.2", - "Pillow==12.1.1", - ] - - extra_setup = [ - "uv pip install " + " ".join(requirements), - ] + extra_setup = ["./tests/scripts/task_install_docs_dependencies.sh"] else: check_gpu() - config_script = f"./tests/scripts/task_config_build_gpu.sh {build_dir}" - scripts = extra_setup + [ - config_script, - f"./tests/scripts/task_build.py --build-dir {build_dir}", - ] + platform_name = "cpu" if cpu else "gpu" + scripts = extra_setup + [f"./tests/scripts/task_build_{platform_name}.sh {build_dir}"] if skip_build: scripts = [] @@ -304,6 +273,7 @@ def docs( tutorial_pattern = os.getenv("TVM_TUTORIAL_EXEC_PATTERN", ".py" if full else "none") env = { + "TVM_TEST_USE_WHEEL": "1", "TVM_TUTORIAL_EXEC_PATTERN": tutorial_pattern, "PYTHON_DOCS_ONLY": "0" if full else "1", "IS_LOCAL": "1", @@ -401,10 +371,12 @@ def generate_command( scripts = [] else: scripts = [ - f"./tests/scripts/task_config_build_{name}.sh {build_dir}", - f"./tests/scripts/task_build.py --build-dir {build_dir}", + f"./tests/scripts/task_build_{name}.sh {build_dir} {int(bool(kwargs.get('cpp')))}", ] + # Direct --tests and interactive Python use the same installed artifacts. + scripts.append("source ./tests/scripts/task_python_install_wheels.sh") + if post_build is not None: scripts += post_build @@ -422,10 +394,7 @@ def generate_command( scripts.extend(script.format(build_dir=build_dir) for script in extra_scripts) docker_env = { - # Need to specify the library path manually or else TVM can't - # determine which build directory to use (i.e. if there are - # multiple copies of libtvm_compiler.so / libtvm_runtime.so laying around) - "TVM_LIBRARY_PATH": str(REPO_ROOT / get_build_dir(name)), + "TVM_TEST_USE_WHEEL": "1", "VERBOSE": "true" if verbose else "false", } @@ -578,7 +547,7 @@ def add_subparser( return subparser -CPP_UNITTEST = ("run c++ unitests", ["./tests/scripts/task_cpp_unittest.sh {build_dir}"]) +CPP_UNITTEST = ("run c++ unitests", ["{build_dir}/tirx-wheel/tirx_cpptest", "{build_dir}/cpptest"]) generated = [ generate_command( diff --git a/tests/scripts/task_build.py b/tests/scripts/task_build.py deleted file mode 100755 index f7bfd558e6..0000000000 --- a/tests/scripts/task_build.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -import argparse -import logging -import multiprocessing -import os -import shutil -import subprocess -import sys -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parent.parent.parent - -if __name__ == "__main__": - logging.basicConfig( - format="[%(filename)s:%(lineno)d %(levelname)s] %(message)s", level=logging.INFO - ) - - parser = argparse.ArgumentParser(description="List pytest nodeids for a folder") - parser.add_argument("--sccache-bucket", required=False, help="sccache bucket name") - parser.add_argument("--sccache-region", required=False, help="sccache region") - parser.add_argument("--build-dir", default="build", help="build folder") - parser.add_argument( - "--cmake-target", action="append", default=[], help="build target (repeatable)" - ) - parser.add_argument( - "--ffi-only", action="store_true", help="prepare the source-pinned FFI package" - ) - parser.add_argument("--debug", required=False, action="store_true", help="build in debug mode") - args = parser.parse_args() - sccache_exe = shutil.which("sccache") - - if args.cmake_target and all( - target in {"standalone_crt", "crttest"} for target in args.cmake_target - ): - logging.info("Skipping standalone_crt build") - exit(0) - - use_sccache = sccache_exe is not None - build_dir = Path(os.getcwd()) / args.build_dir - build_dir = build_dir.relative_to(REPO_ROOT) - env = os.environ.copy() - env["PYTHONPATH"] = os.pathsep.join( - [ - str(REPO_ROOT / "tirx" / "python"), - str(REPO_ROOT / "python"), - os.environ.get("PYTHONPATH", ""), - ] - ) - - if use_sccache: - if args.sccache_bucket and "AWS_ACCESS_KEY_ID" in os.environ: - env["SCCACHE_BUCKET"] = args.sccache_bucket - env["SCCACHE_REGION"] = args.sccache_region if args.sccache_region else "us-west-2" - logging.info(f"Using sccache bucket: {args.sccache_bucket}") - logging.info(f"Using sccache region: {env['SCCACHE_REGION']}") - else: - logging.info("No sccache bucket set, using local cache") - if "SCCACHE_SERVER_PORT" in os.environ: - env["SCCACHE_SERVER_PORT"] = os.getenv("SCCACHE_SERVER_PORT") - env["CXX"] = "/opt/sccache/c++" - env["CC"] = "/opt/sccache/cc" - - else: - if sccache_exe is None: - reason = "'sccache' executable not found" - else: - reason = "<unknown>" - logging.info(f"Not using sccache, reason: {reason}") - - # Build from this checkout's pinned submodule once, then transfer the exact - # Python/native package together with the TVM libraries to test workers. - ffi_commit = subprocess.check_output( - ["git", "-C", str(REPO_ROOT / "3rdparty" / "tvm-ffi"), "rev-parse", "HEAD"], text=True - ).strip() - ffi_identity = f"{ffi_commit} {sys.implementation.cache_tag}" - ffi_marker = REPO_ROOT / "python" / "tvm_ffi" / ".source-commit" - if not ffi_marker.is_file() or ffi_marker.read_text().strip() != ffi_identity: - subprocess.run( - [ - "uv", - "pip", - "install", - "--python", - sys.executable, - "-v", - "--no-deps", - "--reinstall", - "--target=python", - "./3rdparty/tvm-ffi/", - ], - cwd=REPO_ROOT, - env=env, - check=True, - ) - ffi_marker.write_text(ffi_identity + "\n") - if args.ffi_only: - sys.exit(0) - - if use_sccache: - subprocess.run(["sccache", "--start-server"], env=env, check=False) - logging.info("===== sccache stats =====") - subprocess.run(["sccache", "--show-stats"], env=env, check=True) - - executors = int(os.environ.get("CI_NUM_EXECUTORS", 1)) - build_platform = os.environ.get("PLATFORM", None) - - nproc = multiprocessing.cpu_count() - - available_cpus = nproc // executors - num_cpus = max(available_cpus, 1) - - build_type = "Debug" if args.debug else "RelWithDebInfo" - subprocess.run( - ["cmake", "-GNinja", f"-DCMAKE_BUILD_TYPE={build_type}", ".."], - cwd=build_dir, - env=env, - check=True, - ) - - ninja_args = [*args.cmake_target, f"-j{num_cpus}"] - if os.environ.get("VERBOSE", "true").lower() in {"1", "true", "yes"}: - ninja_args.append("-v") - subprocess.run(["cmake", "--build", ".", "--", *ninja_args], cwd=build_dir, env=env, check=True) - - if use_sccache: - logging.info("===== sccache stats =====") - subprocess.run(["sccache", "--show-stats"], env=env, check=True) diff --git a/tests/scripts/task_web_wasm.sh b/tests/scripts/task_build_arm.sh similarity index 69% copy from tests/scripts/task_web_wasm.sh copy to tests/scripts/task_build_arm.sh index aed91a371b..c82dc36698 100755 --- a/tests/scripts/task_web_wasm.sh +++ b/tests/scripts/task_build_arm.sh @@ -18,19 +18,14 @@ set -euxo pipefail -export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" - -# setup tvm-ffi into python folder -uv pip install -v --target=python ./3rdparty/tvm-ffi/ - -rm -rf .emscripten_cache -cd web -export EM_CACHE=`pwd`/.emscripten_cache -make clean -npm install -npm run lint -npm run prepwasm -npm run bundle -npm run test -npm run typedoc -cd .. +ROOTDIR="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOTDIR" +BUILD_DIR="${1:-build}" +CPP_TESTS="${2:-0}" +./tirx/tests/scripts/task_build_arm.sh "$BUILD_DIR" "$CPP_TESTS" +exec ./tests/scripts/task_build_wheels.sh "$BUILD_DIR" "$CPP_TESTS" \ + "-DUSE_SORT=ON" \ + "-DUSE_RPC=ON" \ + "-DCMAKE_CXX_FLAGS=-Werror" \ + "-DUSE_CCACHE=OFF" \ + "-DSUMMARIZE=ON" diff --git a/tests/scripts/task_config_build_arm.sh b/tests/scripts/task_build_cpu.sh similarity index 56% copy from tests/scripts/task_config_build_arm.sh copy to tests/scripts/task_build_cpu.sh index b58b21db34..ec95812cb1 100755 --- a/tests/scripts/task_config_build_arm.sh +++ b/tests/scripts/task_build_cpu.sh @@ -18,14 +18,20 @@ set -euxo pipefail -BUILD_DIR=$1 -mkdir -p "$BUILD_DIR" -cd "$BUILD_DIR" -cp ../cmake/config.cmake . - -echo set\(USE_SORT ON\) >> config.cmake -echo set\(USE_RPC ON\) >> config.cmake -echo set\(USE_LLVM llvm-config-17\) >> config.cmake -echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake -echo set\(USE_CCACHE OFF\) >> config.cmake -echo set\(SUMMARIZE ON\) >> config.cmake +ROOTDIR="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOTDIR" +BUILD_DIR="${1:-build}" +CPP_TESTS="${2:-0}" +./tirx/tests/scripts/task_build_cpu.sh "$BUILD_DIR" "$CPP_TESTS" +exec ./tests/scripts/task_build_wheels.sh "$BUILD_DIR" "$CPP_TESTS" \ + "-DUSE_SORT=ON" \ + "-DUSE_DNNL=ON" \ + "-DUSE_EXAMPLE_NPU_CODEGEN=ON" \ + "-DUSE_EXAMPLE_NPU_RUNTIME=ON" \ + "-DCMAKE_CXX_FLAGS=-Wno-error=range-loop-construct -Wno-error=comment" \ + "-DHIDE_PRIVATE_SYMBOLS=ON" \ + "-DUSE_TENSORFLOW_PATH=/tensorflow" \ + "-DUSE_TFLITE=/opt/tflite" \ + "-DUSE_FLATBUFFERS_PATH=/flatbuffers" \ + "-DUSE_CCACHE=OFF" \ + "-DSUMMARIZE=ON" diff --git a/tests/scripts/task_web_wasm.sh b/tests/scripts/task_build_dependencies.sh old mode 100755 new mode 100644 similarity index 70% copy from tests/scripts/task_web_wasm.sh copy to tests/scripts/task_build_dependencies.sh index aed91a371b..08cdee6633 --- a/tests/scripts/task_web_wasm.sh +++ b/tests/scripts/task_build_dependencies.sh @@ -16,21 +16,9 @@ # specific language governing permissions and limitations # under the License. -set -euxo pipefail - -export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" - -# setup tvm-ffi into python folder -uv pip install -v --target=python ./3rdparty/tvm-ffi/ - -rm -rf .emscripten_cache -cd web -export EM_CACHE=`pwd`/.emscripten_cache -make clean -npm install -npm run lint -npm run prepwasm -npm run bundle -npm run test -npm run typedoc -cd .. +# Source this helper to use the writable build dependency target. +BUILD_DEPS="$(realpath "$1")/wheel-build-deps" +WHEEL_DIR="$PWD/dist" +uv pip install --python python3 --target "$BUILD_DEPS" --no-deps --offline \ + "$WHEEL_DIR"/apache_tvm_tirx-*.whl +export PYTHONPATH="$BUILD_DEPS" diff --git a/tests/scripts/task_config_build_gpu_other.sh b/tests/scripts/task_build_gpu.sh similarity index 57% copy from tests/scripts/task_config_build_gpu_other.sh copy to tests/scripts/task_build_gpu.sh index 0d5d2030c4..ab5ab5c968 100755 --- a/tests/scripts/task_config_build_gpu_other.sh +++ b/tests/scripts/task_build_gpu.sh @@ -16,17 +16,25 @@ # specific language governing permissions and limitations # under the License. -# This file is a compiler test to ensure that runtimes can compile -# correctly, even if they aren't actively tested in the CI. set -euxo pipefail -BUILD_DIR=$1 -mkdir -p "$BUILD_DIR" -cd "$BUILD_DIR" -cp ../cmake/config.cmake . - -echo set\(USE_OPENCL ON\) >> config.cmake -echo set\(USE_ROCM ON\) >> config.cmake -echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake -echo set\(USE_CCACHE OFF\) >> config.cmake -echo set\(SUMMARIZE ON\) >> config.cmake +ROOTDIR="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOTDIR" +BUILD_DIR="${1:-build}" +CPP_TESTS="${2:-0}" +./tirx/tests/scripts/task_build_gpu.sh "$BUILD_DIR" "$CPP_TESTS" +exec ./tests/scripts/task_build_wheels.sh "$BUILD_DIR" "$CPP_TESTS" \ + "-DUSE_CUBLAS=ON" \ + "-DUSE_CUDNN=ON" \ + "-DUSE_CUDA=ON" \ + "-DUSE_RPC=ON" \ + "-DUSE_SORT=ON" \ + "-DUSE_BLAS=openblas" \ + "-DCMAKE_CXX_FLAGS=-Werror" \ + "-DUSE_TENSORRT_CODEGEN=ON" \ + "-DHIDE_PRIVATE_SYMBOLS=ON" \ + "-DUSE_CUTLASS=ON" \ + "-DCMAKE_CUDA_ARCHITECTURES=75" \ + "-DUSE_CLML=ON" \ + "-DUSE_CCACHE=OFF" \ + "-DSUMMARIZE=ON" diff --git a/tests/scripts/task_config_build_cpu.sh b/tests/scripts/task_build_gpu_other.sh similarity index 50% rename from tests/scripts/task_config_build_cpu.sh rename to tests/scripts/task_build_gpu_other.sh index e9d528be11..e848172b4a 100755 --- a/tests/scripts/task_config_build_cpu.sh +++ b/tests/scripts/task_build_gpu_other.sh @@ -16,22 +16,26 @@ # specific language governing permissions and limitations # under the License. +# This file is a compiler test to ensure that runtimes can compile +# correctly, even if they aren't actively tested in the CI. set -euxo pipefail -BUILD_DIR=$1 +BUILD_DIR="${1:-build}" +CPP_TESTS="${2:-0}" +ROOTDIR="$(cd "$(dirname "$0")/../.." && pwd)" mkdir -p "$BUILD_DIR" -cd "$BUILD_DIR" -cp ../cmake/config.cmake . +BUILD_DIR="$(realpath "$BUILD_DIR")" +rm -f "$BUILD_DIR/config.cmake" -echo set\(USE_SORT ON\) >> config.cmake -echo set\(USE_DNNL ON\) >> config.cmake -echo set\(USE_EXAMPLE_NPU_CODEGEN ON\) >> config.cmake -echo set\(USE_EXAMPLE_NPU_RUNTIME ON\) >> config.cmake -echo set\(USE_LLVM \"/usr/bin/llvm-config-17 --link-static\"\) >> config.cmake -echo set\(CMAKE_CXX_FLAGS \"-Wno-error=range-loop-construct -Wno-error=comment\"\) >> config.cmake -echo set\(HIDE_PRIVATE_SYMBOLS ON\) >> config.cmake -echo set\(USE_TENSORFLOW_PATH \"/tensorflow\"\) >> config.cmake -echo set\(USE_TFLITE \"/opt/tflite\"\) >> config.cmake -echo set\(USE_FLATBUFFERS_PATH \"/flatbuffers\"\) >> config.cmake -echo set\(USE_CCACHE OFF\) >> config.cmake -echo set\(SUMMARIZE ON\) >> config.cmake +cd "$ROOTDIR" +# Preserve the separate runtime-only GPU compile check. +uv pip install --python python3 --no-deps --target "$BUILD_DIR/python" ./3rdparty/tvm-ffi +export PYTHONPATH="$BUILD_DIR/python" +if command -v sccache >/dev/null; then + sccache --start-server || true + trap 'sccache --show-stats' EXIT +fi +cmake -S . -B "$BUILD_DIR" -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DUSE_OPENCL=ON -DUSE_ROCM=ON -DCMAKE_CXX_FLAGS=-Werror \ + -DUSE_CCACHE=OFF -DSUMMARIZE=ON +cmake --build "$BUILD_DIR" --target tvm_runtime diff --git a/tests/scripts/task_config_build_gpu_other.sh b/tests/scripts/task_build_wasm.sh similarity index 66% rename from tests/scripts/task_config_build_gpu_other.sh rename to tests/scripts/task_build_wasm.sh index 0d5d2030c4..541d58b444 100755 --- a/tests/scripts/task_config_build_gpu_other.sh +++ b/tests/scripts/task_build_wasm.sh @@ -16,17 +16,16 @@ # specific language governing permissions and limitations # under the License. -# This file is a compiler test to ensure that runtimes can compile -# correctly, even if they aren't actively tested in the CI. set -euxo pipefail -BUILD_DIR=$1 -mkdir -p "$BUILD_DIR" -cd "$BUILD_DIR" -cp ../cmake/config.cmake . - -echo set\(USE_OPENCL ON\) >> config.cmake -echo set\(USE_ROCM ON\) >> config.cmake -echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake -echo set\(USE_CCACHE OFF\) >> config.cmake -echo set\(SUMMARIZE ON\) >> config.cmake +ROOTDIR="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOTDIR" +BUILD_DIR="${1:-build}" +CPP_TESTS="${2:-0}" +./tirx/tests/scripts/task_build_wasm.sh "$BUILD_DIR" "$CPP_TESTS" +exec ./tests/scripts/task_build_wheels.sh "$BUILD_DIR" "$CPP_TESTS" \ + "-DUSE_SORT=ON" \ + "-DCMAKE_CXX_FLAGS=-Werror" \ + "-DHIDE_PRIVATE_SYMBOLS=ON" \ + "-DUSE_CCACHE=OFF" \ + "-DSUMMARIZE=ON" diff --git a/tests/scripts/task_build_wheels.sh b/tests/scripts/task_build_wheels.sh new file mode 100755 index 0000000000..f0012a575e --- /dev/null +++ b/tests/scripts/task_build_wheels.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euxo pipefail + +mkdir -p "${1:-build}" +BUILD_DIR="$(realpath "${1:-build}")" +CPP_TESTS="${2:-0}" +shift 2 +WHEEL_DIR="$PWD/dist" +rm -f "$BUILD_DIR/config.cmake" +if command -v sccache >/dev/null; then + trap 'sccache --show-stats' EXIT +fi +tvm_options=() +if [ "$CPP_TESTS" = "1" ]; then + tvm_options=( '-Cbuild.targets=all;cpptest') +fi +source ./tests/scripts/task_build_dependencies.sh "$BUILD_DIR" +cmake_options=("$@") +if [ "$CPP_TESTS" = "1" ]; then + cmake_options+=(-DUSE_GTEST=ON -DBUILD_TESTING=ON) +fi +cmake_options+=(-DTVM_USE_INSTALLED_TIRX=ON) +cmake_args=$(IFS=';'; echo "${cmake_options[*]}") +uv build --wheel --no-build-isolation --offline --python python3 --out-dir "$WHEEL_DIR" . \ + -Cbuild-dir="$BUILD_DIR" -Ccmake.build-type=RelWithDebInfo \ + -Ccmake.args="$cmake_args" \ + -Cbuild.verbose="${VERBOSE:-true}" "${tvm_options[@]}" +if [ "$CPP_TESTS" = "1" ]; then + "$BUILD_DIR/cpptest" +fi diff --git a/tests/scripts/task_config_build_gpu.sh b/tests/scripts/task_config_build_gpu.sh deleted file mode 100755 index a16847023a..0000000000 --- a/tests/scripts/task_config_build_gpu.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euxo pipefail - -BUILD_DIR=$1 -mkdir -p "$BUILD_DIR" -cd "$BUILD_DIR" -cp ../cmake/config.cmake . - -echo set\(USE_CUBLAS ON\) >> config.cmake -echo set\(USE_CUDNN ON\) >> config.cmake -echo set\(USE_CUDA ON\) >> config.cmake -echo set\(USE_VULKAN ON\) >> config.cmake -echo set\(USE_OPENCL ON\) >> config.cmake -echo set\(USE_OPENCL_GTEST \"/googletest\"\) >> config.cmake -echo set\(USE_LLVM \"/usr/bin/llvm-config-15 --link-static\"\) >> config.cmake -echo set\(USE_RPC ON\) >> config.cmake -echo set\(USE_SORT ON\) >> config.cmake -echo set\(USE_BLAS openblas\) >> config.cmake -echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake -echo set\(USE_TENSORRT_CODEGEN ON\) >> config.cmake -echo set\(USE_CCACHE OFF\) >> config.cmake -echo set\(SUMMARIZE ON\) >> config.cmake -echo set\(HIDE_PRIVATE_SYMBOLS ON\) >> config.cmake -echo set\(USE_CUTLASS ON\) >> config.cmake -echo set\(CMAKE_CUDA_ARCHITECTURES 75\) >> config.cmake -echo set\(USE_CLML ON\) >> config.cmake diff --git a/tests/scripts/task_config_build_wasm.sh b/tests/scripts/task_install_docs_dependencies.sh similarity index 67% copy from tests/scripts/task_config_build_wasm.sh copy to tests/scripts/task_install_docs_dependencies.sh index 11526082e4..2ec999876f 100755 --- a/tests/scripts/task_config_build_wasm.sh +++ b/tests/scripts/task_install_docs_dependencies.sh @@ -18,14 +18,16 @@ set -euxo pipefail -BUILD_DIR=$1 -mkdir -p "$BUILD_DIR" -cd "$BUILD_DIR" -cp ../cmake/config.cmake . - -echo set\(USE_SORT ON\) >> config.cmake -echo set\(USE_LLVM llvm-config-15\) >> config.cmake -echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake -echo set\(HIDE_PRIVATE_SYMBOLS ON\) >> config.cmake -echo set\(USE_CCACHE OFF\) >> config.cmake -echo set\(SUMMARIZE ON\) >> config.cmake +# Additional documentation dependencies for the CPU CI image. +uv pip install \ + 'Sphinx==8.1.3' \ + 'sphinx_autodoc_annotation~=1.0' \ + 'sphinx-gallery==0.20.0' \ + 'sphinx-book-theme==1.1.4' \ + 'pydata-sphinx-theme==0.15.4' \ + 'autodocsumm==0.2.14' \ + 'image==1.5.33' \ + 'matplotlib==3.10.8' \ + 'commonmark==0.9.1' \ + 'docutils==0.21.2' \ + 'Pillow==12.1.1' diff --git a/tests/scripts/task_java_unittest.sh b/tests/scripts/task_java_unittest.sh index 29342c70c8..c072ef26e6 100755 --- a/tests/scripts/task_java_unittest.sh +++ b/tests/scripts/task_java_unittest.sh @@ -18,8 +18,12 @@ set -euxo pipefail -export PYTHONPATH=tirx/python:python -export LD_LIBRARY_PATH="lib:${LD_LIBRARY_PATH:-}" +if [ "${TVM_TEST_USE_WHEEL:-0}" = "1" ]; then + source "$(dirname "$0")/task_python_install_wheels.sh" +else + export PYTHONPATH=tirx/python:python + export LD_LIBRARY_PATH="lib:${LD_LIBRARY_PATH:-}" +fi # to avoid CI CPU thread throttling. export TVM_BIND_THREADS=0 diff --git a/tests/scripts/task_jvm_build.sh b/tests/scripts/task_jvm_build.sh index 468fea4ec6..fede6385f4 100755 --- a/tests/scripts/task_jvm_build.sh +++ b/tests/scripts/task_jvm_build.sh @@ -33,13 +33,32 @@ fi ROOTDIR="$(cd "$(dirname "$0")/../.." && pwd)" TVM_BUILD_PATH="${TVM_BUILD_PATH:-${ROOTDIR}/build}" -TVM_BUILD_PATH="$(realpath "${TVM_BUILD_PATH}")" +if [ "${TVM_TEST_USE_WHEEL:-0}" != "1" ]; then + TVM_BUILD_PATH="$(realpath "${TVM_BUILD_PATH}")" +fi DLPACK_PATH="${DLPACK_PATH:-${ROOTDIR}/3rdparty/tvm-ffi/3rdparty/dlpack}" INCLUDE_FLAGS="-I${ROOTDIR}/include -I${DLPACK_PATH}/include" PKG_CFLAGS="-Wall -O3 ${INCLUDE_FLAGS} -fPIC" PKG_LDFLAGS="" +NATIVE_ARGS=() +if [ "${TVM_TEST_USE_WHEEL:-0}" = "1" ]; then + TVM_INCLUDE_DIR="$(python3 -m tvm_tirx.config --includedir)" + FFI_INCLUDE_DIR="$(python3 -m tvm_ffi.config --includedir)" + TVM_LIB_DIR="$(python3 -m tvm_tirx.config --libdir)" + FFI_LIB_DIR="$(python3 -m tvm_ffi.config --libdir)" + PKG_CFLAGS="-Wall -O3 -fPIC $(python3 -m tvm_tirx.config --cflags) $(python3 -m tvm_ffi.config --cflags)" + PKG_LDFLAGS="$(python3 -m tvm_tirx.config --ldflags --libs) $(python3 -m tvm_ffi.config --ldflags --libs)" + # Each config flag prints one line; Maven expects one compiler/linker argument list. + PKG_LDFLAGS="${PKG_LDFLAGS//$'\n'/ }" + NATIVE_ARGS=( + "-Dtvm.include.dir=${TVM_INCLUDE_DIR}" + "-Dtvm.ffi.include.dir=${FFI_INCLUDE_DIR}" + "-Dtvm.lib.dir=${TVM_LIB_DIR}" + "-Dtvm.ffi.lib.dir=${FFI_LIB_DIR}" + ) +fi if [ "$(uname -s)" = "Darwin" ]; then JVM_PKG_PROFILE="osx-x86_64" @@ -64,5 +83,6 @@ mvn ${GOAL} \ "-Dcflags=${PKG_CFLAGS}" \ "-Dldflags=${PKG_LDFLAGS}" \ "-Dcurrent_libdir=${TVM_BUILD_PATH}" \ + "${NATIVE_ARGS[@]}" \ ${JVM_TEST_ARGS} \ "$@" diff --git a/tests/scripts/task_python_docs.sh b/tests/scripts/task_python_docs.sh index 0a4b88df96..bd9e69393a 100755 --- a/tests/scripts/task_python_docs.sh +++ b/tests/scripts/task_python_docs.sh @@ -18,7 +18,16 @@ set -euxo pipefail -export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" +# The native pipeline opts in; old main-selected jobs retain source imports. +if [ "${TVM_TEST_USE_WHEEL:-0}" = "1" ]; then + source "$(dirname "$0")/task_python_install_wheels.sh" + export TVM_DOCS_USE_WHEEL=1 +else + export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" + unset TVM_DOCS_USE_WHEEL + uv pip install -v --target=python ./3rdparty/tvm-ffi/ + uv pip install --no-deps apache-tvm-ffi-orcjit==0.1.2 +fi # to avoid CI CPU thread throttling. export TVM_BIND_THREADS=0 @@ -47,9 +56,6 @@ sphinx_precheck() { clean_files echo "PreCheck sphinx doc generation WARNINGS.." - # setup tvm-ffi into python folder - uv pip install -v --target=python ./3rdparty/tvm-ffi/ - pushd docs make clean TVM_TUTORIAL_EXEC_PATTERN=none make html 2>&1 | tee /tmp/$$.log.txt @@ -129,15 +135,10 @@ clean_files find . -type f -path "*.log" | xargs rm -f find . -type f -path "*.pyc" | xargs rm -f -# setup tvm-ffi into python folder -uv pip install -v --target=python ./3rdparty/tvm-ffi/ - -# LLVMModule JIT execution, needed by the tutorials this build runs. -uv pip install --no-deps apache-tvm-ffi-orcjit==0.1.2 cd docs -PYTHONPATH=$(pwd)/../tirx/python:$(pwd)/../python make htmldepoly SPHINXOPTS='-j auto' |& tee /tmp/$$.log.txt +make htmldepoly SPHINXOPTS='-j auto' |& tee /tmp/$$.log.txt if grep -E "failed to execute|Segmentation fault" < /tmp/$$.log.txt; then echo "Some of sphinx-gallery item example failed to execute." exit 1 diff --git a/tests/scripts/task_python_install_wheels.sh b/tests/scripts/task_python_install_wheels.sh new file mode 100644 index 0000000000..c435a95f44 --- /dev/null +++ b/tests/scripts/task_python_install_wheels.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Source this file so subsequent Python invocations use the installed wheels. +# Docker defaults PYTHONPATH to the checkout; replace it with the explicit target. +unset PYTHONPATH TVM_LIBRARY_PATH TVM_FFI_LIBRARY_PATH TVM_TIRX_LIBRARY_PATH +export PYTHONNOUSERSITE=1 +export TVM_HOME="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +TVM_PYTHON_TARGET="$TVM_HOME/.local/python" + +# Match the explicit prerequisite in the legacy Python test setup. +uv pip install --python python3 --target "$TVM_PYTHON_TARGET" --no-deps apache-tvm-ffi-orcjit==0.1.2 +uv pip install --python python3 --target "$TVM_PYTHON_TARGET" --no-deps --offline --reinstall \ + "$TVM_HOME"/dist/apache_tvm-*.whl \ + "$TVM_HOME"/dist/apache_tvm_ffi-*.whl \ + "$TVM_HOME"/dist/apache_tvm_tirx-*.whl +export PYTHONPATH="$TVM_PYTHON_TARGET" diff --git a/tests/scripts/task_python_unittest.sh b/tests/scripts/task_python_unittest.sh index 9b319b7a67..5f12809894 100755 --- a/tests/scripts/task_python_unittest.sh +++ b/tests/scripts/task_python_unittest.sh @@ -18,15 +18,23 @@ set -euxo pipefail -export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" -export PYTEST_ADDOPTS="${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" +if [ "${TVM_TEST_USE_WHEEL:-0}" = "1" ]; then + source "$(dirname "$0")/task_python_install_wheels.sh" +else + export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" + + # Source-mode callers provision FFI from the pinned submodule. + uv pip install --python python3 --no-deps --target=python ./3rdparty/tvm-ffi/ -# Reuse the exact source-pinned FFI package transferred by the native build. -python3 tests/scripts/task_build.py --ffi-only + # LLVMModule JIT execution. Installed here as well as in the CI image because a docker/ + # change does not reach the test containers in the same run (they use the pinned tag). + # --no-deps keeps the submodule-built tvm-ffi above from being replaced by a PyPI build. + uv pip install --no-deps apache-tvm-ffi-orcjit==0.1.2 -# LLVMModule JIT execution. Installed here as well as in the CI image because a docker/ -# change does not reach the test containers in the same run (they use the pinned tag). -# --no-deps keeps the submodule-built tvm-ffi above from being replaced by a PyPI build. -uv pip install --no-deps apache-tvm-ffi-orcjit==0.1.2 +fi +export PYTEST_ADDOPTS="${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" -python3 -m pytest -vvs -n auto -o faulthandler_timeout=120 -m "${TVM_TEST_MARKER:-not gpu}" tests/python tirx/tests/python +if [ "$#" -eq 0 ]; then + set -- tests/python tirx/tests/python +fi +python3 -m pytest -vvs -n auto -o faulthandler_timeout=120 -m "${TVM_TEST_MARKER:-not gpu}" "$@" diff --git a/tests/scripts/task_python_unittest_gpuonly.sh b/tests/scripts/task_python_unittest_gpuonly.sh index c025bd6675..2e12d621f8 100755 --- a/tests/scripts/task_python_unittest_gpuonly.sh +++ b/tests/scripts/task_python_unittest_gpuonly.sh @@ -31,9 +31,6 @@ export TVM_TEST_TARGETS='cuda;metal;rocm;nvptx' # only minimal functionality of Vulkan runtime. export TVM_TEST_TARGETS='{"kind":"vulkan","from_device":0}' -export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" -export PYTEST_ADDOPTS="${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" - -python3 -m pytest -vvs -n auto -m "${TVM_TEST_MARKER}" \ +./tests/scripts/task_python_unittest.sh \ tirx/tests/python/backend/vulkan/codegen/test_vulkan.py \ tirx/tests/python/backend/vulkan/test_vulkan_module_export.py diff --git a/tests/scripts/task_web_wasm.sh b/tests/scripts/task_web_wasm.sh index aed91a371b..4573aab846 100755 --- a/tests/scripts/task_web_wasm.sh +++ b/tests/scripts/task_web_wasm.sh @@ -18,10 +18,14 @@ set -euxo pipefail -export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" +if [ "${TVM_TEST_USE_WHEEL:-0}" = "1" ]; then + source "$(dirname "$0")/task_python_install_wheels.sh" +else + export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" -# setup tvm-ffi into python folder -uv pip install -v --target=python ./3rdparty/tvm-ffi/ + # setup tvm-ffi into python folder + uv pip install -v --target=python ./3rdparty/tvm-ffi/ +fi rm -rf .emscripten_cache cd web diff --git a/tirx/pyproject.toml b/tirx/pyproject.toml index db8d5c071c..8c58b29d81 100644 --- a/tirx/pyproject.toml +++ b/tirx/pyproject.toml @@ -55,6 +55,9 @@ dependencies = [ "packaging", ] +[project.scripts] +tvm-tirx-config = "tvm_tirx.config:__main__" + [project.optional-dependencies] torch = ["torch"] cuda = ["cuda-bindings"] diff --git a/tirx/python/tvm_tirx/__init__.py b/tirx/python/tvm_tirx/__init__.py index efbdb05a5e..d583e2964f 100644 --- a/tirx/python/tvm_tirx/__init__.py +++ b/tirx/python/tvm_tirx/__init__.py @@ -18,108 +18,125 @@ # pylint: disable=redefined-builtin, wildcard-import """TVM: Open Deep Learning Compiler Stack.""" -import multiprocessing -import os import sys -# ffi module must load first -from tvm_ffi import register_object, register_global_func, get_global_func +from typing import TYPE_CHECKING -# top-level alias -from .libinfo import __version__ -from .base import _RUNTIME_ONLY -# tvm.runtime -from .runtime import Object -from .runtime._tensor import device, cpu, cuda, opencl, vulkan, metal -from .runtime._tensor import vpi, rocm, ext_dev, hexagon -from .runtime import DataType, DataTypeCode +def _is_config_mode() -> bool: + """Recognize the config CLI before loading Python or native TVM components.""" + if sys.argv[0].endswith("tvm-tirx-config"): + return True + argv = getattr(sys, "orig_argv", []) + return any( + arg == "-m" and argv[i + 1 : i + 2] == ["tvm_tirx.config"] for i, arg in enumerate(argv) + ) -# tvm.error -from . import error -# tvm.ir -from .ir import IRModule, transform -from . import ir +if TYPE_CHECKING or not _is_config_mode(): + import multiprocessing + import os + import sys -# tvm.script — must be imported before any dialect package so that -# tvm.script.register_dialect is reachable when dialect __init__.py files run. -from . import script + # ffi module must load first + from tvm_ffi import register_object, register_global_func, get_global_func -# tvm.tirx — registers itself via tvm.script.register_dialect in its __init__ -from . import tirx -from .ir import prim + # top-level alias + from .libinfo import __version__ + from .base import _RUNTIME_ONLY -# tvm.backend — owns backend Python load hooks -from . import backend + # tvm.runtime + from .runtime import Object + from .runtime._tensor import device, cpu, cuda, opencl, vulkan, metal + from .runtime._tensor import vpi, rocm, ext_dev, hexagon + from .runtime import DataType, DataTypeCode -# tvm.target -from . import target -from .target import device_from_target + # tvm.error + from . import error -# Shared code generation helpers -from . import codegen + # tvm.ir + from .ir import IRModule, transform + from . import ir -# others -from . import sym + # tvm.script — must be imported before any dialect package so that + # tvm.script.register_dialect is reachable when dialect __init__.py files run. + from . import script -# support infra -from . import support + # tvm.tirx — registers itself via tvm.script.register_dialect in its __init__ + from . import tirx + from .ir import prim -# Side-effect imports: register CUDA/ROCm FFI callbacks at TVM startup -from .support import rocm as _rocm, nvcc as _nvcc + # tvm.backend — owns backend Python load hooks + from . import backend -# NOTE: This file should be python2 compatible so we can -# raise proper error message when user run the package using -# an older version of the python + # tvm.target + from . import target + from .target import device_from_target + # Shared code generation helpers + from . import codegen -def _should_print_backtrace(): - in_pytest = "PYTEST_CURRENT_TEST" in os.environ - tvm_backtrace = os.environ.get("TVM_BACKTRACE", "0") + # others + from . import sym - try: - tvm_backtrace = bool(int(tvm_backtrace)) - except ValueError: - raise ValueError(f"invalid value for TVM_BACKTRACE {tvm_backtrace}, please set to 0 or 1.") + # support infra + from . import support + + # Side-effect imports: register CUDA/ROCm FFI callbacks at TVM startup + from .support import rocm as _rocm, nvcc as _nvcc - return in_pytest or tvm_backtrace + # NOTE: This file should be python2 compatible so we can + # raise proper error message when user run the package using + # an older version of the python + def _should_print_backtrace(): + in_pytest = "PYTEST_CURRENT_TEST" in os.environ + tvm_backtrace = os.environ.get("TVM_BACKTRACE", "0") -def tvm_wrap_excepthook(exception_hook): - """Wrap given excepthook with TVM additional work.""" + try: + tvm_backtrace = bool(int(tvm_backtrace)) + except ValueError: + raise ValueError( + f"invalid value for TVM_BACKTRACE {tvm_backtrace}, please set to 0 or 1." + ) - def wrapper(exctype, value, trbk): - """Clean subprocesses when TVM is interrupted.""" - if exctype is error.DiagnosticError and not _should_print_backtrace(): - # TODO(@jroesch): consider moving to C++? - print("note: run with `TVM_BACKTRACE=1` environment variable to display a backtrace.") - else: - exception_hook(exctype, value, trbk) + return in_pytest or tvm_backtrace - if hasattr(multiprocessing, "active_children"): - # pylint: disable=not-callable - for p in multiprocessing.active_children(): - p.terminate() + def tvm_wrap_excepthook(exception_hook): + """Wrap given excepthook with TVM additional work.""" - return wrapper + def wrapper(exctype, value, trbk): + """Clean subprocesses when TVM is interrupted.""" + if exctype is error.DiagnosticError and not _should_print_backtrace(): + # TODO(@jroesch): consider moving to C++? + print( + "note: run with `TVM_BACKTRACE=1` environment variable to display a backtrace." + ) + else: + exception_hook(exctype, value, trbk) + if hasattr(multiprocessing, "active_children"): + # pylint: disable=not-callable + for p in multiprocessing.active_children(): + p.terminate() -sys.excepthook = tvm_wrap_excepthook(sys.excepthook) + return wrapper -# Autoload loads built-in and out-of-tree backends. Out-of-tree extensions opt -# into being loaded automatically at ``import tvm`` time by declaring an entry -# point in the ``tvm.backends`` group: -# [project.entry-points."tvm.backends"] tvm_foo = "tvm_foo:_autoload". -# Autoload can be disabled via ``TVM_DEVICE_BACKEND_AUTOLOAD=0``. -from .backend._autoload_backends import _autoload_backends + sys.excepthook = tvm_wrap_excepthook(sys.excepthook) + + # Autoload loads built-in and out-of-tree backends. Out-of-tree extensions opt + # into being loaded automatically at ``import tvm`` time by declaring an entry + # point in the ``tvm.backends`` group: + # [project.entry-points."tvm.backends"] tvm_foo = "tvm_foo:_autoload". + # Autoload can be disabled via ``TVM_DEVICE_BACKEND_AUTOLOAD=0``. + from .backend._autoload_backends import _autoload_backends -_autoload_backends() + _autoload_backends() -# Load the optional ORC JIT extension for its global FFI registrations. LLVM -# function lookup reports a targeted installation error if this import fails; -# importing TVM itself remains silent for AOT-only workflows. -try: - import tvm_ffi_orcjit as _tvm_ffi_orcjit -except Exception: # pylint: disable=broad-exception-caught - pass + # Load the optional ORC JIT extension for its global FFI registrations. LLVM + # function lookup reports a targeted installation error if this import fails; + # importing TVM itself remains silent for AOT-only workflows. + try: + import tvm_ffi_orcjit as _tvm_ffi_orcjit + except Exception: # pylint: disable=broad-exception-caught + pass diff --git a/tirx/python/tvm_tirx/config.py b/tirx/python/tvm_tirx/config.py new file mode 100644 index 0000000000..ea69971fbc --- /dev/null +++ b/tirx/python/tvm_tirx/config.py @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Lightweight compiler and linker configuration for the TVM runtime.""" + +import argparse +import os +import sys +from pathlib import Path + + +def _runtime_root() -> Path: + """Locate this package without importing its native code.""" + return Path(__file__).resolve().parent + + +def _find_include() -> str: + package = _runtime_root() + for path in (package / "include", package.parent.parent / "include"): + if (path / "tvm" / "runtime").is_dir(): + return str(path.resolve()) + raise RuntimeError("Cannot find TVM runtime headers") + + +def _library_paths() -> list[Path]: + package = _runtime_root() + source = package.parent.parent + paths = [package / "lib", source / "build" / "lib", source / "lib"] + if source.name == "tirx": + paths.append(source.parent / "build" / "lib") + if os.environ.get("TVM_TIRX_LIBRARY_PATH"): + paths.insert(0, Path(os.environ["TVM_TIRX_LIBRARY_PATH"])) + for name in ("LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH", "PATH"): + paths.extend(Path(value) for value in os.environ.get(name, "").split(os.pathsep) if value) + return paths + + +def _find_runtime() -> str: + filename = {"win32": "tvm_runtime.dll", "darwin": "libtvm_runtime.dylib"}.get( + sys.platform, "libtvm_runtime.so" + ) + for directory in _library_paths(): + path = directory / filename + if path.is_file(): + return str(path.resolve()) + raise RuntimeError("Cannot find libtvm_runtime") + + +def __main__() -> None: + """Print paths and flags using the same conventions as tvm_ffi.config.""" + parser = argparse.ArgumentParser(description="Get TIRx runtime build configuration") + for name, help_text in ( + ("includedir", "Print include directory"), + ("libdir", "Print library directory"), + ("libfiles", "Fully qualified library filenames"), + ("libs", "Libraries to be linked"), + ("cflags", "Print C flags"), + ("cxxflags", "Print C++ flags"), + ("ldflags", "Print linker flags"), + ): + parser.add_argument(f"--{name}", action="store_true", help=help_text) + args = parser.parse_args() + if len(sys.argv) == 1: + parser.print_help() + return + if args.includedir: + print(_find_include()) + if args.libdir or args.libfiles or args.libs or args.ldflags: + library = Path(_find_runtime()) + link_file = library.with_suffix(".lib") if sys.platform == "win32" else library + if args.libdir: + print(library.parent) + if args.libfiles: + print(link_file) + if args.cxxflags: + print(f"-I{_find_include()} -std=c++20") + if args.cflags: + print(f"-I{_find_include()}") + if args.libs: + print(link_file if sys.platform == "win32" else "-ltvm_runtime") + if args.ldflags and sys.platform != "win32": + print(f"-L{library.parent}") + + +if __name__ == "__main__": + __main__() diff --git a/tirx/tests/python/runtime/test_tirx_config.py b/tirx/tests/python/runtime/test_tirx_config.py new file mode 100644 index 0000000000..cab4876265 --- /dev/null +++ b/tirx/tests/python/runtime/test_tirx_config.py @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""The config command must work without importing native runtime dependencies.""" + +import os +import subprocess +import sys +from pathlib import Path + +import tvm_tirx +from tvm_tirx import config + + +def test_config_without_native_imports(tmp_path): + # A startup hook rejects native loads and dependencies even when they are installed. + (tmp_path / "sitecustomize.py").write_text( + "import sys\n" + "class BlockNative:\n" + " def find_spec(self, fullname, path=None, target=None):\n" + " if fullname.split('.')[0] in ('tvm_ffi', 'torch', 'numpy') or " + "fullname in ('tvm.base', 'tvm.runtime', 'tvm.backend', 'tvm.driver', 'tvm_tirx.base'):\n" + " raise AssertionError(fullname)\n" + "sys.meta_path.insert(0, BlockNative())\n" + "def audit(event, args):\n" + " if event == 'ctypes.dlopen': raise AssertionError(args)\n" + "sys.addaudithook(audit)\n" + ) + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join( + ( + str(tmp_path), + str(Path(tvm_tirx.__file__).parent.parent), + ) + ) + command = [sys.executable, "-m", "tvm_tirx.config"] + include = subprocess.check_output([*command, "--includedir"], env=env, text=True).strip() + assert (Path(include) / "tvm" / "runtime").is_dir() + library = subprocess.check_output([*command, "--libfiles"], env=env, text=True).strip() + assert Path(library).is_file() + flags = subprocess.check_output( + [*command, "--cflags", "--cxxflags", "--libs", "--ldflags"], env=env, text=True + ) + assert f"-I{include}" in flags + assert "-std=c++20" in flags + + +def test_runtime_library_path_fallback(tmp_path, monkeypatch): + library = tmp_path / Path(config._find_runtime()).name + library.touch() + monkeypatch.setattr(config, "_runtime_root", lambda: tmp_path / "tvm_tirx") + monkeypatch.delenv("LD_LIBRARY_PATH", raising=False) + monkeypatch.delenv("DYLD_LIBRARY_PATH", raising=False) + monkeypatch.setenv("PATH", str(tmp_path)) + assert config._find_runtime() == str(library) + + +if __name__ == "__main__": + import tvm_tirx.testing + + tvm_tirx.testing.main() diff --git a/tests/scripts/task_config_build_arm.sh b/tirx/tests/scripts/task_build_arm.sh similarity index 70% rename from tests/scripts/task_config_build_arm.sh rename to tirx/tests/scripts/task_build_arm.sh index b58b21db34..cdbaf5b125 100755 --- a/tests/scripts/task_config_build_arm.sh +++ b/tirx/tests/scripts/task_build_arm.sh @@ -18,14 +18,12 @@ set -euxo pipefail -BUILD_DIR=$1 -mkdir -p "$BUILD_DIR" -cd "$BUILD_DIR" -cp ../cmake/config.cmake . - -echo set\(USE_SORT ON\) >> config.cmake -echo set\(USE_RPC ON\) >> config.cmake -echo set\(USE_LLVM llvm-config-17\) >> config.cmake -echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake -echo set\(USE_CCACHE OFF\) >> config.cmake -echo set\(SUMMARIZE ON\) >> config.cmake +ROOTDIR="$(cd "$(dirname "$0")/../../.." && pwd)" +cd "$ROOTDIR" +BUILD_DIR="${1:-build}" +CPP_TESTS="${2:-0}" +exec ./tirx/tests/scripts/task_build_wheels.sh "$BUILD_DIR" "$CPP_TESTS" \ + "-DUSE_LLVM=llvm-config-17" \ + "-DCMAKE_CXX_FLAGS=-Werror" \ + "-DUSE_CCACHE=OFF" \ + "-DSUMMARIZE=ON" diff --git a/tests/scripts/task_web_wasm.sh b/tirx/tests/scripts/task_build_cpu.sh similarity index 67% copy from tests/scripts/task_web_wasm.sh copy to tirx/tests/scripts/task_build_cpu.sh index aed91a371b..541b9e90af 100755 --- a/tests/scripts/task_web_wasm.sh +++ b/tirx/tests/scripts/task_build_cpu.sh @@ -18,19 +18,13 @@ set -euxo pipefail -export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" - -# setup tvm-ffi into python folder -uv pip install -v --target=python ./3rdparty/tvm-ffi/ - -rm -rf .emscripten_cache -cd web -export EM_CACHE=`pwd`/.emscripten_cache -make clean -npm install -npm run lint -npm run prepwasm -npm run bundle -npm run test -npm run typedoc -cd .. +ROOTDIR="$(cd "$(dirname "$0")/../../.." && pwd)" +cd "$ROOTDIR" +BUILD_DIR="${1:-build}" +CPP_TESTS="${2:-0}" +exec ./tirx/tests/scripts/task_build_wheels.sh "$BUILD_DIR" "$CPP_TESTS" \ + "-DUSE_LLVM=/usr/bin/llvm-config-17 --link-static" \ + "-DCMAKE_CXX_FLAGS=-Wno-error=range-loop-construct -Wno-error=comment" \ + "-DHIDE_PRIVATE_SYMBOLS=ON" \ + "-DUSE_CCACHE=OFF" \ + "-DSUMMARIZE=ON" diff --git a/tests/scripts/task_web_wasm.sh b/tirx/tests/scripts/task_build_dependencies.sh old mode 100755 new mode 100644 similarity index 60% copy from tests/scripts/task_web_wasm.sh copy to tirx/tests/scripts/task_build_dependencies.sh index aed91a371b..b059a27ed4 --- a/tests/scripts/task_web_wasm.sh +++ b/tirx/tests/scripts/task_build_dependencies.sh @@ -16,21 +16,14 @@ # specific language governing permissions and limitations # under the License. -set -euxo pipefail +# Source this helper to use the writable build dependency target. +BUILD_DEPS="$(realpath "$1")/wheel-build-deps" +WHEEL_DIR="$PWD/dist" +rm -rf "$WHEEL_DIR" "$BUILD_DEPS" +uv build --wheel --python python3 --out-dir "$WHEEL_DIR" ./3rdparty/tvm-ffi +uv pip install --python python3 --target "$BUILD_DEPS" \ + "scikit-build-core>=0.11" "setuptools-scm>=8" "mlc-z3-static>=4.16.0" +uv pip install --python python3 --target "$BUILD_DEPS" --no-deps --offline \ + "$WHEEL_DIR"/apache_tvm_ffi-*.whl -export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" - -# setup tvm-ffi into python folder -uv pip install -v --target=python ./3rdparty/tvm-ffi/ - -rm -rf .emscripten_cache -cd web -export EM_CACHE=`pwd`/.emscripten_cache -make clean -npm install -npm run lint -npm run prepwasm -npm run bundle -npm run test -npm run typedoc -cd .. +export PYTHONPATH="$BUILD_DEPS" diff --git a/tests/scripts/task_web_wasm.sh b/tirx/tests/scripts/task_build_gpu.sh similarity index 62% copy from tests/scripts/task_web_wasm.sh copy to tirx/tests/scripts/task_build_gpu.sh index aed91a371b..6cd3e7de54 100755 --- a/tests/scripts/task_web_wasm.sh +++ b/tirx/tests/scripts/task_build_gpu.sh @@ -18,19 +18,18 @@ set -euxo pipefail -export PYTHONPATH="$(pwd)/tirx/python:$(pwd)/python" - -# setup tvm-ffi into python folder -uv pip install -v --target=python ./3rdparty/tvm-ffi/ - -rm -rf .emscripten_cache -cd web -export EM_CACHE=`pwd`/.emscripten_cache -make clean -npm install -npm run lint -npm run prepwasm -npm run bundle -npm run test -npm run typedoc -cd .. +ROOTDIR="$(cd "$(dirname "$0")/../../.." && pwd)" +cd "$ROOTDIR" +BUILD_DIR="${1:-build}" +CPP_TESTS="${2:-0}" +exec ./tirx/tests/scripts/task_build_wheels.sh "$BUILD_DIR" "$CPP_TESTS" \ + "-DUSE_CUDA=ON" \ + "-DUSE_VULKAN=ON" \ + "-DUSE_OPENCL=ON" \ + "-DUSE_OPENCL_GTEST=/googletest" \ + "-DUSE_LLVM=/usr/bin/llvm-config-15 --link-static" \ + "-DCMAKE_CXX_FLAGS=-Werror" \ + "-DHIDE_PRIVATE_SYMBOLS=ON" \ + "-DCMAKE_CUDA_ARCHITECTURES=75" \ + "-DUSE_CCACHE=OFF" \ + "-DSUMMARIZE=ON" diff --git a/tests/scripts/task_config_build_wasm.sh b/tirx/tests/scripts/task_build_wasm.sh similarity index 69% rename from tests/scripts/task_config_build_wasm.sh rename to tirx/tests/scripts/task_build_wasm.sh index 11526082e4..233bdd4ff6 100755 --- a/tests/scripts/task_config_build_wasm.sh +++ b/tirx/tests/scripts/task_build_wasm.sh @@ -18,14 +18,13 @@ set -euxo pipefail -BUILD_DIR=$1 -mkdir -p "$BUILD_DIR" -cd "$BUILD_DIR" -cp ../cmake/config.cmake . - -echo set\(USE_SORT ON\) >> config.cmake -echo set\(USE_LLVM llvm-config-15\) >> config.cmake -echo set\(CMAKE_CXX_FLAGS -Werror\) >> config.cmake -echo set\(HIDE_PRIVATE_SYMBOLS ON\) >> config.cmake -echo set\(USE_CCACHE OFF\) >> config.cmake -echo set\(SUMMARIZE ON\) >> config.cmake +ROOTDIR="$(cd "$(dirname "$0")/../../.." && pwd)" +cd "$ROOTDIR" +BUILD_DIR="${1:-build}" +CPP_TESTS="${2:-0}" +exec ./tirx/tests/scripts/task_build_wheels.sh "$BUILD_DIR" "$CPP_TESTS" \ + "-DUSE_LLVM=llvm-config-15" \ + "-DCMAKE_CXX_FLAGS=-Werror" \ + "-DHIDE_PRIVATE_SYMBOLS=ON" \ + "-DUSE_CCACHE=OFF" \ + "-DSUMMARIZE=ON" diff --git a/tirx/tests/scripts/task_build_wheels.sh b/tirx/tests/scripts/task_build_wheels.sh new file mode 100755 index 0000000000..e051a5d1b2 --- /dev/null +++ b/tirx/tests/scripts/task_build_wheels.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euxo pipefail + +mkdir -p "${1:-build}" +BUILD_DIR="$(realpath "${1:-build}")" +CPP_TESTS="${2:-0}" +shift 2 +WHEEL_DIR="$PWD/dist" +TIRX_BUILD="$BUILD_DIR/tirx-wheel" + +if command -v sccache >/dev/null; then + sccache --start-server || true + sccache --show-stats + trap 'sccache --show-stats' EXIT +fi + +source ./tirx/tests/scripts/task_build_dependencies.sh "$BUILD_DIR" + +# TIRx owns the core native runtime. Build it with the same platform configuration. +mkdir -p "$TIRX_BUILD" +rm -f "$TIRX_BUILD/config.cmake" +tirx_options=() +if [ "$CPP_TESTS" = "1" ]; then + tirx_options=( '-Cbuild.targets=all;tirx_cpptest') +fi +cmake_options=("$@") +if [ "$CPP_TESTS" = "1" ]; then + cmake_options+=(-DUSE_GTEST=ON -DBUILD_TESTING=ON) +fi +cmake_options+=("-DTVM_OPENCL_HEADERS_ROOT=$PWD/3rdparty/OpenCL-Headers") +cmake_args=$(IFS=';'; echo "${cmake_options[*]}") +uv build --wheel --no-build-isolation --offline --python python3 --out-dir "$WHEEL_DIR" ./tirx \ + -Cbuild-dir="$TIRX_BUILD" -Ccmake.build-type=RelWithDebInfo \ + -Ccmake.args="$cmake_args" \ + -Cbuild.verbose="${VERBOSE:-true}" "${tirx_options[@]}" +if [ "$CPP_TESTS" = "1" ]; then + "$TIRX_BUILD/tirx_cpptest" +fi
