This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch jenkins-wheel-test-artifacts in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 7a8784af3595f2ed91bf4d5291e2ad97229cf1a8 Merge: c748eef52e f216560d1e Author: Tianqi Chen <[email protected]> AuthorDate: Wed Oct 14 23:02:33 2026 +0000 [CI] Adapt wheel artifacts to native Jenkins pipelines Build only TVM with the image-provided backend and dependencies, and install its exact wheel for Python consumers. Keep C++ tests in the packaging build tree, pass native parallel limits through Docker, and preserve specialized runtime and Java artifacts. .github/actions/setup/action.yml | 6 +- .github/workflows/nightly_docker_update.yml | 24 - .github/workflows/publish_wheel.yml | 6 +- .github/workflows/tvmbot.yml | 2 +- ci/README.md | 15 +- ci/{scripts => }/github/__init__.py | 0 ci/{scripts/jenkins => github}/git_utils.py | 0 ci/{scripts => }/github/github_tvmbot.py | 12 +- ci/jenkins/README.md | 91 ++- ci/jenkins/data.py | 26 +- ci/jenkins/generate.py | 226 ------- ci/jenkins/generated/arm_jenkinsfile.groovy | 526 ++------------- .../{wasm_jenkinsfile.groovy => common.groovy} | 339 ++++------ ci/jenkins/generated/cpu_jenkinsfile.groovy | 590 ++--------------- ci/jenkins/generated/docker_jenkinsfile.groovy | 696 +++----------------- ci/jenkins/generated/gpu_jenkinsfile.groovy | 728 +++------------------ ci/jenkins/generated/wasm_jenkinsfile.groovy | 527 ++------------- .../scripts}/determine_docker_images.py | 7 +- .../scripts}/git_change_docker.sh | 0 .../jenkins => jenkins/scripts}/git_skip_ci.py | 51 +- .../scripts}/git_skip_ci_globs.py | 2 +- .../jenkins => jenkins/scripts}/http_utils.py | 0 ci/{scripts/jenkins => jenkins/scripts}/retry.sh | 0 ci/{scripts/jenkins => jenkins/scripts}/s3.py | 45 +- .../scripts}/should_rebuild_docker.py | 19 +- .../jenkins/scripts/show_node_info.sh | 0 ci/jenkins/templates/arm_jenkinsfile.groovy.j2 | 34 - ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 | 50 -- ci/jenkins/templates/docker_jenkinsfile.groovy.j2 | 202 ------ ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 | 148 ----- ci/jenkins/templates/utils/Build.groovy.j2 | 43 -- ci/jenkins/templates/utils/Prepare.groovy.j2 | 276 -------- ci/jenkins/templates/utils/base.groovy.j2 | 110 ---- ci/jenkins/templates/utils/macros.j2 | 169 ----- ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 | 37 -- ci/scripts/jenkins/cmd_utils.py | 107 --- ci/scripts/jenkins/git_change_docs.sh | 39 -- ci/scripts/jenkins/open_docker_update_pr.py | 214 ------ ci/scripts/jenkins/should_run_slow_tests.py | 85 --- ci/{scripts/package => wheel}/README.md | 0 .../package => wheel}/build-environment.yaml | 0 .../manylinux_build_libtvm_runtime_cuda.sh | 2 +- .../windows_build_libtvm_runtime_cuda.bat | 4 +- docker/bash.sh | 2 +- docker/dev_common.sh | 2 +- docs/contribute/ci.rst | 2 +- docs/contribute/testing.rst | 2 +- pyproject.toml | 2 +- tests/python/ci/test_ci.py | 122 +--- tests/python/ci/test_jenkins_helpers.py | 80 +++ tests/python/ci/test_tvmbot.py | 6 +- tests/python/ci/test_utils.py | 4 +- tests/python/ci/test_wheel_artifacts.py | 40 +- tests/scripts/ci.py | 23 +- tests/scripts/release/gather_prs.py | 6 +- tests/scripts/release/make_notes.py | 5 - tests/scripts/setup-pytest-env.sh | 2 +- tests/scripts/task_python_docs.sh | 3 - tests/scripts/task_python_install_wheels.sh | 36 +- tests/scripts/task_python_unittest.sh | 5 - tests/scripts/task_python_unittest_gpuonly.sh | 4 +- tests/scripts/task_web_wasm.sh | 6 +- 62 files changed, 794 insertions(+), 5016 deletions(-) diff --cc ci/jenkins/README.md index dd6b652bee,1294228054..4b98759224 --- a/ci/jenkins/README.md +++ b/ci/jenkins/README.md @@@ -52,4 -91,6 +91,30 @@@ No job path or controller reconfigurati # Infrastructure - While all TVM tests are contained within the apache/tvm repository, the infrastructure used to run the tests is donated by the TVM Community. To encourage collaboration, the configuration for TVM's CI infrastructure is stored in a public GitHub repository. TVM community members are encouraged to contribute improvements. The configuration, along with documentation of TVM's CI infrastructure, is in the [tlc-pack/ci](https://github.com/tlc-pack/ci) repo. + The community's infrastructure configuration and documentation live in + [tlc-pack/ci](https://github.com/tlc-pack/ci). Coordinate external callers when + changing any retained script interface. ++ ++## Wheel build and test artifacts ++ ++Normal builds use `uv build --wheel --no-build-isolation --offline` with the ++existing platform CMake configuration. `uv build` builds only TVM; it does not ++resolve runtime dependencies. CPU, ARM and Wasm host builds request `all;cpptest` ++and run C++ tests in that same native tree before uploading anything. Python ++consumers install the exact TVM wheel with `--no-deps --offline`, clear checkout ++path overrides, and verify the package and native-library paths. FFI comes from ++the image. GPU Java tests also receive their existing native runtime libraries; ++the separate runtime-only compile check and Emscripten runtime build remain. ++ ++The selected images must already provide the requirements in `pyproject.toml` ++(including scikit-build-core, setuptools-scm and compatible apache-tvm-ffi), the ++platform runtime dependencies, and the test dependencies. The pinned image must ++be checked before rollout: the current image recipe does not provision all these ++wheel prerequisites. Missing dependencies are errors, not permission for a test ++stage to fetch or build replacements. No image publication or Jenkins deployment ++is performed by this change. ++ ++`CMAKE_BUILD_PARALLEL_LEVEL` is passed through Docker as an environment variable. ++An explicit value wins; otherwise builds use at least one CPU and divide the ++node CPU count by `CI_NUM_EXECUTORS`. Both the wheel backend and specialized ++native commands use that bound without an additional parallel flag. diff --cc ci/jenkins/data.py index ebf29fc91b,fd75107e7d..83773ca2ea --- a/ci/jenkins/data.py +++ b/ci/jenkins/data.py @@@ -31,28 -31,28 +31,10 @@@ data-driven artifact lists import sys files_to_stash = { - # Exact TVM and pinned FFI wheels, installed by Python test/docs consumers. - # Executables and build files needed to run c++ tests - "cpptest": ["build/cpptest", "build/build.ninja", "build/CMakeFiles/rules.ninja"], - # runtime files - "tvm_runtime": ["build/lib/libtvm_runtime.so", "build/config.cmake"], - # compiler files - "tvm_lib": [ - "build/lib/libtvm_compiler.so", - "build/lib/libtvm_runtime.so", - "build/lib/libtvm_ffi.so", - "build/lib/libtvm_runtime_cuda.so", - "build/lib/libtvm_runtime_vulkan.so", - "build/lib/libtvm_runtime_opencl.so", - "build/lib/libtvm_runtime_rocm.so", - "build/lib/libtvm_runtime_extra.so", - "build/libtvm_allvisible.so", - "build/config.cmake", - ], - # gpu related compiler files - "tvm_lib_gpu_extra": [ - "build/3rdparty/libflash_attn/src/libflash_attn.so", - "build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so", - ], ++ # Exact TVM wheels, installed by Python test/docs consumers. + "wheels": ["build/wheels"], + # Maven's native assembly still consumes these paths directly. + "tvm_runtime": ["build/lib/libtvm_runtime.so", "build/lib/libtvm_ffi.so"], - # compiler files - "tvm_lib": [ - "build/lib/libtvm_compiler.so", - "build/lib/libtvm_runtime.so", - "build/lib/libtvm_ffi.so", - "build/lib/libtvm_runtime_cuda.so", - "build/lib/libtvm_runtime_vulkan.so", - "build/lib/libtvm_runtime_opencl.so", - "build/lib/libtvm_runtime_rocm.so", - "build/lib/libtvm_runtime_extra.so", - "build/libtvm_allvisible.so", - "build/config.cmake", - ], - # gpu related compiler files - "tvm_lib_gpu_extra": [ - "build/3rdparty/libflash_attn/src/libflash_attn.so", - "build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so", - ], } diff --cc ci/jenkins/generated/arm_jenkinsfile.groovy index 1670f37db7,e70ce07525..9e5e4a93b8 --- a/ci/jenkins/generated/arm_jenkinsfile.groovy +++ b/ci/jenkins/generated/arm_jenkinsfile.groovy @@@ -14,498 -14,53 +14,52 @@@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - // -*- mode: groovy -*- - // Licensed to the Apache Software Foundation (ASF) under one - // or more contributor license agreements. See the NOTICE file - // distributed with this work for additional information - // regarding copyright ownership. The ASF licenses this file - // to you under the Apache License, Version 2.0 (the - // "License"); you may not use this file except in compliance - // with the License. You may obtain a copy of the License at - // - // http://www.apache.org/licenses/LICENSE-2.0 - // - // Unless required by applicable law or agreed to in writing, - // software distributed under the License is distributed on an - // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - // KIND, either express or implied. See the License for the - // specific language governing permissions and limitations - // under the License. - - // Jenkins pipeline - // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - - // Docker env used for testing - // Different image may have different version tag - // because some of them are more stable than anoter. - // - // Docker images are maintained by PMC, cached in dockerhub - // and remains relatively stable over the time. - // Flow for upgrading docker env(need commiter) - // - // - Send PR to upgrade build script in the repo - // - Build the new docker image - // - Tag the docker image with a new version and push to a binary cache. - // - Update the version in the Jenkinsfile, send a PR - // - Fix any issues wrt to the new image version in the PR - // - Merge the PR and now we are in new version - // - Tag the new version as the lates - // - Periodically cleanup the old versions on local workers - // - - // ============================= IMPORTANT NOTE ============================= - // This file is generated by 'jenkins/generate.py'. Do not edit this file directly! - // Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with - // 'python3 jenkins/generate.py' - // Note: This timestamp is here to ensure that updates to the Jenkinsfile are - // always rebased on main before merging: - // Generated at 2026-10-14T21:26:39.010162 - - import org.jenkinsci.plugins.pipeline.modeldefinition.Utils - // These are set at runtime from data in ci/jenkins/docker-images.yml, update - // image tags in that file - ci_gpu = '' - ci_cpu = '' - ci_minimal = '' - ci_wasm = '' - ci_cortexm = '' - ci_arm = '' - ci_riscv = '' - - // Parameters to allow overriding (in Jenkins UI), the images - // to be used by a given build. When provided, they take precedence - // over default values above. - properties([ - parameters([ - string(name: 'ci_arm_param', defaultValue: ''), - string(name: 'ci_cpu_param', defaultValue: ''), - string(name: 'ci_gpu_param', defaultValue: ''), - string(name: 'ci_wasm_param', defaultValue: ''), - ]) - ]) - - // Placeholders for newly built Docker image names (if rebuild_docker_images - // is used) - built_ci_arm = null; - built_ci_cpu = null; - built_ci_gpu = null; - built_ci_wasm = null; - - // Global variable assigned during Sanity Check that holds the sha1 which should be - // merged into the PR in all branches. - upstream_revision = null - - // command to start a docker container - docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' - docker_build = 'docker/build.sh' - // timeout in minutes - max_time = 180 - rebuild_docker_images = false - - s3_bucket = 'tvm-jenkins-artifacts-prod' - s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" - - // Jenkins script root directory - jenkins_scripts_root = "ci/scripts/jenkins" - - - // General note: Jenkins has limits on the size of a method (or top level code) - // that are pretty strict, so most usage of groovy methods in these templates - // are purely to satisfy the JVM - def per_exec_ws(folder) { - return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder - } - - // initialize source codes - def init_git() { - retry(5) { - checkout scm - } - - // Add more info about job node - sh ( - script: './tests/scripts/task_show_node_info.sh', - label: 'Show executor node info', - ) - - // Determine merge commit to use for all stages - if (env.CHANGE_TARGET) { - // This is a PR build, so merge with the latest of the PR's target branch - // (e.g. main or a release branch like v0.25.0). - merge_with_target() - } else { - // This is a branch build (main or a release branch). Only set - // upstream_revision to HEAD and skip merging to avoid a race with another - // commit merged to the branch. - update_upstream_revision("HEAD") - } - - sh( - script: """ - set -eux - . ${jenkins_scripts_root}/retry.sh - retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0 - """, - label: 'Update git submodules', - ) - checkout_trusted_files() - } - - def update_upstream_revision(git_ref) { - if (upstream_revision == null) { - upstream_revision = sh( - script: "git log -1 ${git_ref} --format=\'%H\'", - label: 'Determine upstream revision', - returnStdout: true, - ).trim() - } - } - - def merge_with_target() { - def target = env.CHANGE_TARGET - sh ( - script: "git fetch origin ${target}", - label: "Fetch target branch ${target}", - ) - update_upstream_revision("FETCH_HEAD") - sh ( - script: "git -c user.name=TVM-Jenkins -c [email protected] merge ${upstream_revision}", - label: "Merge to origin/${target}" - ) - } - - def docker_init(image) { - // Clear out all Docker images that aren't going to be used - sh( - script: """ - set -eux - docker image ls --all - IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}') - - echo -e "Found images:\\n\$IMAGES" - echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; } - - docker image ls --all - """, - label: 'Clean old Docker images', - ) - - if (image.contains("amazonaws.com")) { - // If this string is in the image name it's from ECR and needs to be pulled - // with the right credentials - ecr_pull(image) - } else { - sh( - script: """ - set -eux - . ${jenkins_scripts_root}/retry.sh - retry 5 docker pull ${image} - """, - label: 'Pull docker image', - ) - } - } - - def ecr_pull(full_name) { - aws_account_id = sh( - returnStdout: true, - script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', - label: 'Get AWS ID' - ).trim() - - try { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2', - "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { - sh( - script: ''' - set -eux - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO - ''', - label: 'Log in to ECR' - ) - sh( - script: """ - set -eux - . ${jenkins_scripts_root}/retry.sh - retry 5 docker pull ${full_name} - """, - label: 'Pull image from ECR' - ) - } - } finally { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2', - "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { - sh( - script: 'docker logout $AWS_ECR_REPO', - label: 'Clean up login credentials' - ) + // readTrusted pins trusted revisions, but rejects helper edits in untrusted PRs. + // In that case use the PR target, pinning the same commit for all source merges. + def helper + String target_revision = null + try { + helper = readTrusted('ci/jenkins/generated/common.groovy') + } catch (hudson.AbortException | java.io.FileNotFoundException ex) { + if (!env.CHANGE_TARGET) throw ex + echo 'Loading shared pipeline from the pinned PR target: ' + ex.toString() + def from_target = { label -> + node(label) { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/pipeline") { + // No PR-owned executable or credentials run during this bootstrap. + deleteDir() + retry(5) { checkout scm } + // Quote the branch as data and fetch only from the canonical repository. + def target = env.CHANGE_TARGET.replace("'", "'\"'\"'") + sh(script: "git fetch https://github.com/apache/tvm.git 'refs/heads/${target}'", + label: 'Fetch trusted pipeline target') + def revision = sh(script: 'git rev-parse FETCH_HEAD', returnStdout: true).trim() + def source = sh(script: "git show ${revision}:ci/jenkins/generated/common.groovy", + returnStdout: true) + return [revision, source] + } } } - } - - def should_skip_slow_tests(pr_number) { - withCredentials([string( - credentialsId: 'tvm-bot-jenkins-reader', - variable: 'GITHUB_TOKEN', - )]) { - // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests - result = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'", - label: 'Check if CI should run slow tests', - ) - } - return result == 0 - } - - def cancel_previous_build() { - // cancel previous build if it is not on main. - if (env.BRANCH_NAME != 'main') { - def buildNumber = env.BUILD_NUMBER as int - // Milestone API allows us to cancel previous build - // with the same milestone number - if (buildNumber > 1) milestone(buildNumber - 1) - milestone(buildNumber) - } - } - - def is_last_build() { - // check whether it is last build + def pinned try { - return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number - } catch (Throwable ex) { - echo 'Error during check is_last_build ' + ex.toString() - return false - } - } - - def checkout_trusted_files() { - // trust everything from branch builds - if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { - return; - } - - // trust peoople listed in CONTRIBUTING.md - grep_code = sh( - returnStatus: true, - script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'", - label: 'Check if change is from a contributor', - ) - - if (grep_code == 1) { - // Any scripts that run on the bare host and not inside a Docker container - // (especially those that access secrets) should be checked out here so - // only trusted versions are used in CI - sh( - script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.", - label: 'Check out trusted files', - ) - } - } - - def should_skip_ci(pr_number) { - if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { - // never skip CI on build sourced from a branch - return false - } - glob_skip_ci_code = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_skip_ci_globs.py", - label: 'Check if CI should be skipped due to changed files', - ) - if (glob_skip_ci_code == 0) { - return true - } - withCredentials([string( - credentialsId: 'tvm-bot-jenkins-reader', - variable: 'GITHUB_TOKEN', - )]) { - // Exit code of 1 means run full CI (or the script had an error, so run - // full CI just in case). Exit code of 0 means skip CI. - git_skip_ci_code = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'", - label: 'Check if CI should be skipped', - ) - } - return git_skip_ci_code == 0 - } - - def prepare(node_type) { - stage('Prepare') { - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") { - init_git() - - if (env.DETERMINE_DOCKER_IMAGES == 'yes') { - sh( - script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cpu ci_gpu ci_wasm ", - label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images', - ) - // Pull image names from the results of should_rebuild_docker.py - ci_arm = sh( - script: "cat .docker-image-names/ci_arm", - label: "Find docker image name for ci_arm", - returnStdout: true, - ).trim() - ci_cpu = sh( - script: "cat .docker-image-names/ci_cpu", - label: "Find docker image name for ci_cpu", - returnStdout: true, - ).trim() - ci_gpu = sh( - script: "cat .docker-image-names/ci_gpu", - label: "Find docker image name for ci_gpu", - returnStdout: true, - ).trim() - ci_wasm = sh( - script: "cat .docker-image-names/ci_wasm", - label: "Find docker image name for ci_wasm", - returnStdout: true, - ).trim() - } - - ci_arm = params.ci_arm_param ?: ci_arm - ci_cpu = params.ci_cpu_param ?: ci_cpu - ci_gpu = params.ci_gpu_param ?: ci_gpu - ci_wasm = params.ci_wasm_param ?: ci_wasm - - sh (script: """ - echo "Docker images being used in this build:" - echo " ci_arm = ${ci_arm}" - echo " ci_cpu = ${ci_cpu}" - echo " ci_gpu = ${ci_gpu}" - echo " ci_wasm = ${ci_wasm}" - """, label: 'Docker image names') - - is_docs_only_build = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_change_docs.sh", - label: 'Check for docs only changes', - ) - skip_ci = should_skip_ci(env.CHANGE_ID) - skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) - rebuild_docker_images = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_change_docker.sh", - label: 'Check for any docker changes', - ) - - if (skip_ci) { - // Don't rebuild when skipping CI - rebuild_docker_images = false - } - } - } + pinned = from_target('CPU-SMALL-SPOT') + } catch (Exception failure) { + pinned = from_target('CPU-SMALL') } + target_revision = pinned[0] + helper = pinned[1] } - def python_unittest(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", - label: 'Run Python unit tests', - ) - } + // Evaluation is outside the read/fetch fallback: script errors must propagate. + ci = evaluate(helper) + if (target_revision != null) ci.upstream_revision = target_revision + ci.start() - // Keep compiler caching and executor limits identical for wheels and the - // separate runtime-only compile check. Build commands remain at their callers. - def build_environment() { - return ''' - set -eux - export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} )) - if [ "$CMAKE_BUILD_PARALLEL_LEVEL" -lt 1 ]; then export CMAKE_BUILD_PARALLEL_LEVEL=1; fi - if command -v sccache >/dev/null; then - export CC=/opt/sccache/cc CXX=/opt/sccache/c++ - if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then - export SCCACHE_BUCKET=tvm-sccache-prod SCCACHE_REGION=us-west-2 - fi - sccache --start-server || true - sccache --show-stats - trap "sccache --show-stats" EXIT - fi - ''' - } - - def wheel_build(image, cpp_tests = false) { - def commands = build_environment() + ''' - rm -rf build/wheels - uv build --wheel --python python3 --out-dir build/wheels ./3rdparty/tvm-ffi - uv build --wheel --python python3 --out-dir build/wheels . \ - -Ccmake.build-type=RelWithDebInfo -Cbuild.verbose="${VERBOSE:-true}" - '''.stripIndent().trim() - if (cpp_tests) { - commands += " -Ccmake.define.USE_GTEST=ON -Cbuild.targets=all\\;cpptest" - // cpptest links the same native tree that scikit-build just packaged. - commands += "\n./tests/scripts/task_cpp_unittest.sh build" - } - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${image} bash -c '${commands}'", - label: 'Build wheel artifacts' + (cpp_tests ? ' and run C++ tests' : ''), + ci.build('ARM-GRAVITON3', 'BUILD: arm', 'tvm/build-arm', images.ci_arm, 'arm') { + sh( + script: "${docker_run} ${images.ci_arm} ./tests/scripts/task_config_build_arm.sh build", + label: 'Create ARM CMake config', ) - ci.cmake_build(images.ci_arm, 'build') - ci.make_cpp_tests(images.ci_arm, 'build') - ci.upload('arm', '--bundle tvm_lib --bundle cpptest') ++ ci.wheel_build(images.ci_arm, true) ++ ci.upload('arm', '--bundle wheels') } - - cancel_previous_build() - - try { - prepare('CPU-SMALL-SPOT') - } catch(Exception ex) { - prepare('CPU-SMALL') - } - def run_build(node_type) { - if (!skip_ci && is_docs_only_build != 1) { - echo 'Begin running node_type ' + node_type - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-arm") { - init_git() - docker_init(ci_arm) - timeout(time: max_time, unit: 'MINUTES') { - - withEnv([ - 'PLATFORM=arm', - ], { - sh ( - script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build", - label: 'Create ARM CMake config', - ) - wheel_build(ci_arm, true) - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/arm --bundle wheels", - label: 'Upload artifacts to S3', - ) - }) - } - } - } - echo 'End running node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('BUILD: arm') - } - } - def build() { - stage('Build') { - try { - run_build('ARM-GRAVITON3-SPOT') - } catch (hudson.AbortException abortEx) { - echo "Received normal AbortException, exit now. Details:" + abortEx.toString() - throw abortEx - } catch (Throwable ex) { - echo 'Exception during SPOT run ' + ex.toString() - if (is_last_build()) { - // retry if we are currently at last build - // mark the current stage as success - // and try again via on demand node - echo 'Retry on-demand given it is last build' - currentBuild.result = 'SUCCESS' - run_build('ARM-GRAVITON3') - } else { - echo 'Exit since it is not last build' - throw ex - } - } - } - } - build() diff --cc ci/jenkins/generated/common.groovy index cd9861e7f6,cd8324b5f2..cebd9a8d08 --- a/ci/jenkins/generated/common.groovy +++ b/ci/jenkins/generated/common.groovy @@@ -411,99 -290,56 +290,69 @@@ def python_unittest(image) ) } - // Keep compiler caching and executor limits identical for wheels and the - // separate runtime-only compile check. Build commands remain at their callers. -def make_cpp_tests(image, build_dir) { - sh ( - script: """ - set -eux - ${docker_run} ${image} python3 ./tests/scripts/task_build.py \ - --sccache-bucket tvm-sccache-prod \ - --sccache-region us-west-2 \ - --cmake-target cpptest \ - --build-dir ${build_dir} - """, - label: 'Make C++ tests', - ) ++// Share only the compiler/cache environment; build commands stay at their callers. +def build_environment() { + return ''' + set -eux - export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} )) - if [ "$CMAKE_BUILD_PARALLEL_LEVEL" -lt 1 ]; then export CMAKE_BUILD_PARALLEL_LEVEL=1; fi + if command -v sccache >/dev/null; then + export CC=/opt/sccache/cc CXX=/opt/sccache/c++ + if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then + export SCCACHE_BUCKET=tvm-sccache-prod SCCACHE_REGION=us-west-2 + fi + sccache --start-server || true + sccache --show-stats + trap "sccache --show-stats" EXIT + fi + ''' } -def cmake_build(image, path) { - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir ${path}", - label: 'Run CMake build', - ) ++// Preserve explicit caller limits, otherwise divide this node among its executors. ++def build_docker() { ++ return '''set -eu ++ parallel=${CMAKE_BUILD_PARALLEL_LEVEL:-$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} ))} ++ if [ "$parallel" -lt 1 ]; then parallel=1; fi ++ ''' + "${docker_run} --env CMAKE_BUILD_PARALLEL_LEVEL=\"\$parallel\" " + } -def cpp_unittest(image) { - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh", - label: 'Run C++ tests', - ) ++ +def wheel_build(image, cpp_tests = false) { ++ // uv build never resolves runtime dependencies. No isolation plus offline uses ++ // only the backend and FFI already provisioned in the image. + def commands = build_environment() + ''' + rm -rf build/wheels - uv build --wheel --python python3 --out-dir build/wheels ./3rdparty/tvm-ffi - uv build --wheel --python python3 --out-dir build/wheels . \ - -Ccmake.build-type=RelWithDebInfo -Cbuild.verbose="${VERBOSE:-true}" ++ uv build --wheel --no-build-isolation --offline --python python3 --out-dir build/wheels . \ ++ -Cbuild-dir=build -Ccmake.build-type=RelWithDebInfo -Cbuild.verbose="${VERBOSE:-true}" + '''.stripIndent().trim() + if (cpp_tests) { + commands += " -Ccmake.define.USE_GTEST=ON -Cbuild.targets=all\\;cpptest" - // cpptest links the same native tree that scikit-build just packaged. + commands += "\n./tests/scripts/task_cpp_unittest.sh build" + } - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${image} bash -c '${commands}'", - label: 'Build wheel artifacts' + (cpp_tests ? ' and run C++ tests' : ''), - ) ++ sh(script: build_docker() + "${image} bash -c '${commands}'", ++ label: 'Build TVM wheel' + (cpp_tests ? ' and run C++ tests' : '')) } - cancel_previous_build() - - try { + // Preparation retries Exceptions unconditionally; Errors and cancellation of the + // previous build are deliberately outside this fallback. + def start() { + cancel_previous_build() + try { prepare('CPU-SMALL-SPOT') - } catch(Exception ex) { - prepare('CPU-SMALL') - } - def run_build(node_type) { - if (!skip_ci && is_docs_only_build != 1) { - echo 'Begin running node_type ' + node_type - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") { - init_git() - docker_init(ci_wasm) - timeout(time: max_time, unit: 'MINUTES') { - - withEnv([ - 'PLATFORM=wasm', - ], { - sh ( - script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", - label: 'Create WASM CMake config', - ) - wheel_build(ci_wasm, true) - sh ( - script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh", - label: 'Run WASM lint and tests', - ) - }) - } - } - } - echo 'End running node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('BUILD: WASM') + } catch (Exception ex) { + prepare('CPU-SMALL') } } - def build() { + + // A failed build command is final. Infrastructure failures retry only while + // this is the newest build; tests intentionally also retry AbortException. + def build(label, name, folder, image, platform, body) { stage('Build') { try { - run_build('CPU-SPOT') - } catch (hudson.AbortException abortEx) { - echo "Received normal AbortException, exit now. Details:" + abortEx.toString() - throw abortEx + run_on_node("${label}-SPOT", name, folder, image, ["PLATFORM=${platform}"], body) + } catch (hudson.AbortException ex) { + echo 'Received normal AbortException, exit now. Details:' + ex.toString() + throw ex } catch (Throwable ex) { - echo 'Exception during SPOT run ' + ex.toString() - if (is_last_build()) { - // retry if we are currently at last build - // mark the current stage as success - // and try again via on demand node - echo 'Retry on-demand given it is last build' - currentBuild.result = 'SUCCESS' - run_build('CPU') - } else { - echo 'Exit since it is not last build' - throw ex + retry_latest(ex) { + run_on_node(label, name, folder, image, ["PLATFORM=${platform}"], body) } } } diff --cc ci/jenkins/generated/cpu_jenkinsfile.groovy index 56255dc5fb,e252832c65..c91c3571a5 --- a/ci/jenkins/generated/cpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy @@@ -14,559 -14,63 +14,61 @@@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - // -*- mode: groovy -*- - - // Licensed to the Apache Software Foundation (ASF) under one - // or more contributor license agreements. See the NOTICE file - // distributed with this work for additional information - // regarding copyright ownership. The ASF licenses this file - // to you under the Apache License, Version 2.0 (the - // "License"); you may not use this file except in compliance - // with the License. You may obtain a copy of the License at - // - // http://www.apache.org/licenses/LICENSE-2.0 - // - // Unless required by applicable law or agreed to in writing, - // software distributed under the License is distributed on an - // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - // KIND, either express or implied. See the License for the - // specific language governing permissions and limitations - // under the License. - - // Jenkins pipeline - // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - - // Docker env used for testing - // Different image may have different version tag - // because some of them are more stable than anoter. - // - // Docker images are maintained by PMC, cached in dockerhub - // and remains relatively stable over the time. - // Flow for upgrading docker env(need commiter) - // - // - Send PR to upgrade build script in the repo - // - Build the new docker image - // - Tag the docker image with a new version and push to a binary cache. - // - Update the version in the Jenkinsfile, send a PR - // - Fix any issues wrt to the new image version in the PR - // - Merge the PR and now we are in new version - // - Tag the new version as the lates - // - Periodically cleanup the old versions on local workers - // - - // ============================= IMPORTANT NOTE ============================= - // This file is generated by 'jenkins/generate.py'. Do not edit this file directly! - // Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with - // 'python3 jenkins/generate.py' - // Note: This timestamp is here to ensure that updates to the Jenkinsfile are - // always rebased on main before merging: - // Generated at 2026-10-14T21:26:39.031061 - - import org.jenkinsci.plugins.pipeline.modeldefinition.Utils - // These are set at runtime from data in ci/jenkins/docker-images.yml, update - // image tags in that file - ci_gpu = '' - ci_cpu = '' - ci_minimal = '' - ci_wasm = '' - ci_cortexm = '' - ci_arm = '' - ci_riscv = '' - - // Parameters to allow overriding (in Jenkins UI), the images - // to be used by a given build. When provided, they take precedence - // over default values above. - properties([ - parameters([ - string(name: 'ci_arm_param', defaultValue: ''), - string(name: 'ci_cpu_param', defaultValue: ''), - string(name: 'ci_gpu_param', defaultValue: ''), - string(name: 'ci_wasm_param', defaultValue: ''), - ]) - ]) - - // Placeholders for newly built Docker image names (if rebuild_docker_images - // is used) - built_ci_arm = null; - built_ci_cpu = null; - built_ci_gpu = null; - built_ci_wasm = null; - - // Global variable assigned during Sanity Check that holds the sha1 which should be - // merged into the PR in all branches. - upstream_revision = null - - // command to start a docker container - docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' - docker_build = 'docker/build.sh' - // timeout in minutes - max_time = 180 - rebuild_docker_images = false - - s3_bucket = 'tvm-jenkins-artifacts-prod' - s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" - - // Jenkins script root directory - jenkins_scripts_root = "ci/scripts/jenkins" - - - // General note: Jenkins has limits on the size of a method (or top level code) - // that are pretty strict, so most usage of groovy methods in these templates - // are purely to satisfy the JVM - def per_exec_ws(folder) { - return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder - } - - // initialize source codes - def init_git() { - retry(5) { - checkout scm - } - - // Add more info about job node - sh ( - script: './tests/scripts/task_show_node_info.sh', - label: 'Show executor node info', - ) - - // Determine merge commit to use for all stages - if (env.CHANGE_TARGET) { - // This is a PR build, so merge with the latest of the PR's target branch - // (e.g. main or a release branch like v0.25.0). - merge_with_target() - } else { - // This is a branch build (main or a release branch). Only set - // upstream_revision to HEAD and skip merging to avoid a race with another - // commit merged to the branch. - update_upstream_revision("HEAD") - } - - sh( - script: """ - set -eux - . ${jenkins_scripts_root}/retry.sh - retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0 - """, - label: 'Update git submodules', - ) - checkout_trusted_files() - } - - def update_upstream_revision(git_ref) { - if (upstream_revision == null) { - upstream_revision = sh( - script: "git log -1 ${git_ref} --format=\'%H\'", - label: 'Determine upstream revision', - returnStdout: true, - ).trim() - } - } - - def merge_with_target() { - def target = env.CHANGE_TARGET - sh ( - script: "git fetch origin ${target}", - label: "Fetch target branch ${target}", - ) - update_upstream_revision("FETCH_HEAD") - sh ( - script: "git -c user.name=TVM-Jenkins -c [email protected] merge ${upstream_revision}", - label: "Merge to origin/${target}" - ) - } - - def docker_init(image) { - // Clear out all Docker images that aren't going to be used - sh( - script: """ - set -eux - docker image ls --all - IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}') - - echo -e "Found images:\\n\$IMAGES" - echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; } - - docker image ls --all - """, - label: 'Clean old Docker images', - ) - - if (image.contains("amazonaws.com")) { - // If this string is in the image name it's from ECR and needs to be pulled - // with the right credentials - ecr_pull(image) - } else { - sh( - script: """ - set -eux - . ${jenkins_scripts_root}/retry.sh - retry 5 docker pull ${image} - """, - label: 'Pull docker image', - ) - } - } - - def ecr_pull(full_name) { - aws_account_id = sh( - returnStdout: true, - script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', - label: 'Get AWS ID' - ).trim() - - try { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2', - "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { - sh( - script: ''' - set -eux - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO - ''', - label: 'Log in to ECR' - ) - sh( - script: """ - set -eux - . ${jenkins_scripts_root}/retry.sh - retry 5 docker pull ${full_name} - """, - label: 'Pull image from ECR' - ) - } - } finally { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2', - "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { - sh( - script: 'docker logout $AWS_ECR_REPO', - label: 'Clean up login credentials' - ) - } - } - } - - def should_skip_slow_tests(pr_number) { - withCredentials([string( - credentialsId: 'tvm-bot-jenkins-reader', - variable: 'GITHUB_TOKEN', - )]) { - // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests - result = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'", - label: 'Check if CI should run slow tests', - ) - } - return result == 0 - } - - def cancel_previous_build() { - // cancel previous build if it is not on main. - if (env.BRANCH_NAME != 'main') { - def buildNumber = env.BUILD_NUMBER as int - // Milestone API allows us to cancel previous build - // with the same milestone number - if (buildNumber > 1) milestone(buildNumber - 1) - milestone(buildNumber) - } - } - - def is_last_build() { - // check whether it is last build - try { - return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number - } catch (Throwable ex) { - echo 'Error during check is_last_build ' + ex.toString() - return false - } - } - - def checkout_trusted_files() { - // trust everything from branch builds - if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { - return; - } - - // trust peoople listed in CONTRIBUTING.md - grep_code = sh( - returnStatus: true, - script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'", - label: 'Check if change is from a contributor', - ) - - if (grep_code == 1) { - // Any scripts that run on the bare host and not inside a Docker container - // (especially those that access secrets) should be checked out here so - // only trusted versions are used in CI - sh( - script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.", - label: 'Check out trusted files', - ) - } - } - - def should_skip_ci(pr_number) { - if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { - // never skip CI on build sourced from a branch - return false - } - glob_skip_ci_code = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_skip_ci_globs.py", - label: 'Check if CI should be skipped due to changed files', - ) - if (glob_skip_ci_code == 0) { - return true - } - withCredentials([string( - credentialsId: 'tvm-bot-jenkins-reader', - variable: 'GITHUB_TOKEN', - )]) { - // Exit code of 1 means run full CI (or the script had an error, so run - // full CI just in case). Exit code of 0 means skip CI. - git_skip_ci_code = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'", - label: 'Check if CI should be skipped', - ) - } - return git_skip_ci_code == 0 - } - - def prepare(node_type) { - stage('Prepare') { - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") { - init_git() - - if (env.DETERMINE_DOCKER_IMAGES == 'yes') { - sh( - script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cpu ci_gpu ci_wasm ", - label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images', - ) - // Pull image names from the results of should_rebuild_docker.py - ci_arm = sh( - script: "cat .docker-image-names/ci_arm", - label: "Find docker image name for ci_arm", - returnStdout: true, - ).trim() - ci_cpu = sh( - script: "cat .docker-image-names/ci_cpu", - label: "Find docker image name for ci_cpu", - returnStdout: true, - ).trim() - ci_gpu = sh( - script: "cat .docker-image-names/ci_gpu", - label: "Find docker image name for ci_gpu", - returnStdout: true, - ).trim() - ci_wasm = sh( - script: "cat .docker-image-names/ci_wasm", - label: "Find docker image name for ci_wasm", - returnStdout: true, - ).trim() - } - - ci_arm = params.ci_arm_param ?: ci_arm - ci_cpu = params.ci_cpu_param ?: ci_cpu - ci_gpu = params.ci_gpu_param ?: ci_gpu - ci_wasm = params.ci_wasm_param ?: ci_wasm - - sh (script: """ - echo "Docker images being used in this build:" - echo " ci_arm = ${ci_arm}" - echo " ci_cpu = ${ci_cpu}" - echo " ci_gpu = ${ci_gpu}" - echo " ci_wasm = ${ci_wasm}" - """, label: 'Docker image names') - - is_docs_only_build = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_change_docs.sh", - label: 'Check for docs only changes', - ) - skip_ci = should_skip_ci(env.CHANGE_ID) - skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) - rebuild_docker_images = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_change_docker.sh", - label: 'Check for any docker changes', - ) - - if (skip_ci) { - // Don't rebuild when skipping CI - rebuild_docker_images = false - } - } - } - } - } - def python_unittest(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", - label: 'Run Python unit tests', - ) - } - - // Keep compiler caching and executor limits identical for wheels and the - // separate runtime-only compile check. Build commands remain at their callers. - def build_environment() { - return ''' - set -eux - export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} )) - if [ "$CMAKE_BUILD_PARALLEL_LEVEL" -lt 1 ]; then export CMAKE_BUILD_PARALLEL_LEVEL=1; fi - if command -v sccache >/dev/null; then - export CC=/opt/sccache/cc CXX=/opt/sccache/c++ - if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then - export SCCACHE_BUCKET=tvm-sccache-prod SCCACHE_REGION=us-west-2 - fi - sccache --start-server || true - sccache --show-stats - trap "sccache --show-stats" EXIT - fi - ''' - } - - def wheel_build(image, cpp_tests = false) { - def commands = build_environment() + ''' - rm -rf build/wheels - uv build --wheel --python python3 --out-dir build/wheels ./3rdparty/tvm-ffi - uv build --wheel --python python3 --out-dir build/wheels . \ - -Ccmake.build-type=RelWithDebInfo -Cbuild.verbose="${VERBOSE:-true}" - '''.stripIndent().trim() - if (cpp_tests) { - commands += " -Ccmake.define.USE_GTEST=ON -Cbuild.targets=all\\;cpptest" - // cpptest links the same native tree that scikit-build just packaged. - commands += "\n./tests/scripts/task_cpp_unittest.sh build" - } - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${image} bash -c '${commands}'", - label: 'Build wheel artifacts' + (cpp_tests ? ' and run C++ tests' : ''), - ) - } - - cancel_previous_build() + // readTrusted pins trusted revisions, but rejects helper edits in untrusted PRs. + // In that case use the PR target, pinning the same commit for all source merges. + def helper + String target_revision = null try { - prepare('CPU-SMALL-SPOT') - } catch(Exception ex) { - prepare('CPU-SMALL') - } - def run_build(node_type) { - if (!skip_ci && is_docs_only_build != 1) { - echo 'Begin running node_type ' + node_type - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") { - init_git() - docker_init(ci_cpu) - timeout(time: max_time, unit: 'MINUTES') { - - withEnv([ - 'PLATFORM=cpu', - ], { - sh ( - script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", - label: 'Create CPU CMake config', - ) - wheel_build(ci_cpu, true) - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu --bundle wheels", - label: 'Upload artifacts to S3', - ) - }) - } + helper = readTrusted('ci/jenkins/generated/common.groovy') + } catch (hudson.AbortException | java.io.FileNotFoundException ex) { + if (!env.CHANGE_TARGET) throw ex + echo 'Loading shared pipeline from the pinned PR target: ' + ex.toString() + def from_target = { label -> + node(label) { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/pipeline") { + // No PR-owned executable or credentials run during this bootstrap. + deleteDir() + retry(5) { checkout scm } + // Quote the branch as data and fetch only from the canonical repository. + def target = env.CHANGE_TARGET.replace("'", "'\"'\"'") + sh(script: "git fetch https://github.com/apache/tvm.git 'refs/heads/${target}'", + label: 'Fetch trusted pipeline target') + def revision = sh(script: 'git rev-parse FETCH_HEAD', returnStdout: true).trim() + def source = sh(script: "git show ${revision}:ci/jenkins/generated/common.groovy", + returnStdout: true) + return [revision, source] } } - echo 'End running node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('BUILD: CPU') } - } - def build() { - stage('Build') { - try { - run_build('CPU-SPOT') - } catch (hudson.AbortException abortEx) { - echo "Received normal AbortException, exit now. Details:" + abortEx.toString() - throw abortEx - } catch (Throwable ex) { - echo 'Exception during SPOT run ' + ex.toString() - if (is_last_build()) { - // retry if we are currently at last build - // mark the current stage as success - // and try again via on demand node - echo 'Retry on-demand given it is last build' - currentBuild.result = 'SUCCESS' - run_build('CPU') - } else { - echo 'Exit since it is not last build' - throw ex - } - } + def pinned + try { + pinned = from_target('CPU-SMALL-SPOT') + } catch (Exception failure) { + pinned = from_target('CPU-SMALL') } + target_revision = pinned[0] + helper = pinned[1] } - build() + // Evaluation is outside the read/fetch fallback: script errors must propagate. + ci = evaluate(helper) + if (target_revision != null) ci.upstream_revision = target_revision + ci.start() - - - def run_unittest_CPU(node_type) { - echo 'Begin running on node_type ' + node_type - if (!skip_ci && is_docs_only_build != 1) { - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") { - // NOTE: if exception happens, it will be caught outside - init_git() - docker_init(ci_cpu) - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=cpu', - 'TEST_STEP_NAME=unittest: CPU', - "SKIP_SLOW_TESTS=${skip_slow_tests}"], { - sh( - script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu", - label: 'Download artifacts from S3', - ) - - python_unittest(ci_cpu) - }) - } - } - } - echo 'End running on node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('unittest: CPU') - } + ci.build('CPU', 'BUILD: CPU', 'tvm/build-cpu', images.ci_cpu, 'cpu') { + sh( + script: "${docker_run} ${images.ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", + label: 'Create CPU CMake config', + ) - ci.cmake_build(images.ci_cpu, 'build') - ci.make_cpp_tests(images.ci_cpu, 'build') - ci.upload('cpu', '--bundle tvm_lib --bundle cpptest') ++ ci.wheel_build(images.ci_cpu, true) ++ ci.upload('cpu', '--bundle wheels') } - - def test() { - stage('Test') { - environment { - SKIP_SLOW_TESTS = "${skip_slow_tests}" + stage('Test') { + parallel('unittest: CPU': { + ci.test('CPU-SMALL', 'unittest: CPU', 'tvm/ut-python-cpu', images.ci_cpu, 'cpu') { + ci.download('cpu') - ci.cpp_unittest(images.ci_cpu) + ci.python_unittest(images.ci_cpu) } - parallel( - 'unittest: CPU': { - try { - run_unittest_CPU('CPU-SMALL-SPOT') - } catch (Throwable ex) { - echo 'Exception during SPOT run ' + ex.toString() - if (is_last_build()) { - // retry if at last build - // mark the current stage as success - // and try again via on demand node - echo 'Retry on-demand given it is last build' - currentBuild.result = 'SUCCESS' - run_unittest_CPU('CPU-SMALL') - } else { - echo 'Exit since it is not last build' - throw ex - } - } - }, - ) - } + }) } - test() diff --cc ci/jenkins/generated/gpu_jenkinsfile.groovy index 2421da1881,8c98cd16cb..0f81b529b6 --- a/ci/jenkins/generated/gpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy @@@ -14,613 -14,71 +14,75 @@@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - // -*- mode: groovy -*- - - // Licensed to the Apache Software Foundation (ASF) under one - // or more contributor license agreements. See the NOTICE file - // distributed with this work for additional information - // regarding copyright ownership. The ASF licenses this file - // to you under the Apache License, Version 2.0 (the - // "License"); you may not use this file except in compliance - // with the License. You may obtain a copy of the License at - // - // http://www.apache.org/licenses/LICENSE-2.0 - // - // Unless required by applicable law or agreed to in writing, - // software distributed under the License is distributed on an - // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - // KIND, either express or implied. See the License for the - // specific language governing permissions and limitations - // under the License. - - // Jenkins pipeline - // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - - // Docker env used for testing - // Different image may have different version tag - // because some of them are more stable than anoter. - // - // Docker images are maintained by PMC, cached in dockerhub - // and remains relatively stable over the time. - // Flow for upgrading docker env(need commiter) - // - // - Send PR to upgrade build script in the repo - // - Build the new docker image - // - Tag the docker image with a new version and push to a binary cache. - // - Update the version in the Jenkinsfile, send a PR - // - Fix any issues wrt to the new image version in the PR - // - Merge the PR and now we are in new version - // - Tag the new version as the lates - // - Periodically cleanup the old versions on local workers - // - - // ============================= IMPORTANT NOTE ============================= - // This file is generated by 'jenkins/generate.py'. Do not edit this file directly! - // Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with - // 'python3 jenkins/generate.py' - // Note: This timestamp is here to ensure that updates to the Jenkinsfile are - // always rebased on main before merging: - // Generated at 2026-10-14T21:26:39.068100 - - import org.jenkinsci.plugins.pipeline.modeldefinition.Utils - // These are set at runtime from data in ci/jenkins/docker-images.yml, update - // image tags in that file - ci_gpu = '' - ci_cpu = '' - ci_minimal = '' - ci_wasm = '' - ci_cortexm = '' - ci_arm = '' - ci_riscv = '' - - // Parameters to allow overriding (in Jenkins UI), the images - // to be used by a given build. When provided, they take precedence - // over default values above. - properties([ - parameters([ - string(name: 'ci_arm_param', defaultValue: ''), - string(name: 'ci_cpu_param', defaultValue: ''), - string(name: 'ci_gpu_param', defaultValue: ''), - string(name: 'ci_wasm_param', defaultValue: ''), - ]) - ]) - - // Placeholders for newly built Docker image names (if rebuild_docker_images - // is used) - built_ci_arm = null; - built_ci_cpu = null; - built_ci_gpu = null; - built_ci_wasm = null; - - // Global variable assigned during Sanity Check that holds the sha1 which should be - // merged into the PR in all branches. - upstream_revision = null - - // command to start a docker container - docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' - docker_build = 'docker/build.sh' - // timeout in minutes - max_time = 180 - rebuild_docker_images = false - - s3_bucket = 'tvm-jenkins-artifacts-prod' - s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" - - // Jenkins script root directory - jenkins_scripts_root = "ci/scripts/jenkins" - - - // General note: Jenkins has limits on the size of a method (or top level code) - // that are pretty strict, so most usage of groovy methods in these templates - // are purely to satisfy the JVM - def per_exec_ws(folder) { - return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder - } - - // initialize source codes - def init_git() { - retry(5) { - checkout scm - } - - // Add more info about job node - sh ( - script: './tests/scripts/task_show_node_info.sh', - label: 'Show executor node info', - ) - - // Determine merge commit to use for all stages - if (env.CHANGE_TARGET) { - // This is a PR build, so merge with the latest of the PR's target branch - // (e.g. main or a release branch like v0.25.0). - merge_with_target() - } else { - // This is a branch build (main or a release branch). Only set - // upstream_revision to HEAD and skip merging to avoid a race with another - // commit merged to the branch. - update_upstream_revision("HEAD") - } - - sh( - script: """ - set -eux - . ${jenkins_scripts_root}/retry.sh - retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0 - """, - label: 'Update git submodules', - ) - checkout_trusted_files() - } - - def update_upstream_revision(git_ref) { - if (upstream_revision == null) { - upstream_revision = sh( - script: "git log -1 ${git_ref} --format=\'%H\'", - label: 'Determine upstream revision', - returnStdout: true, - ).trim() - } - } - - def merge_with_target() { - def target = env.CHANGE_TARGET - sh ( - script: "git fetch origin ${target}", - label: "Fetch target branch ${target}", - ) - update_upstream_revision("FETCH_HEAD") - sh ( - script: "git -c user.name=TVM-Jenkins -c [email protected] merge ${upstream_revision}", - label: "Merge to origin/${target}" - ) - } - - def docker_init(image) { - // Clear out all Docker images that aren't going to be used - sh( - script: """ - set -eux - docker image ls --all - IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}') - - echo -e "Found images:\\n\$IMAGES" - echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; } - - docker image ls --all - """, - label: 'Clean old Docker images', - ) - - if (image.contains("amazonaws.com")) { - // If this string is in the image name it's from ECR and needs to be pulled - // with the right credentials - ecr_pull(image) - } else { - sh( - script: """ - set -eux - . ${jenkins_scripts_root}/retry.sh - retry 5 docker pull ${image} - """, - label: 'Pull docker image', - ) - } - } - - def ecr_pull(full_name) { - aws_account_id = sh( - returnStdout: true, - script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', - label: 'Get AWS ID' - ).trim() - - try { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2', - "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { - sh( - script: ''' - set -eux - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO - ''', - label: 'Log in to ECR' - ) - sh( - script: """ - set -eux - . ${jenkins_scripts_root}/retry.sh - retry 5 docker pull ${full_name} - """, - label: 'Pull image from ECR' - ) - } - } finally { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2', - "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { - sh( - script: 'docker logout $AWS_ECR_REPO', - label: 'Clean up login credentials' - ) - } - } - } - - def should_skip_slow_tests(pr_number) { - withCredentials([string( - credentialsId: 'tvm-bot-jenkins-reader', - variable: 'GITHUB_TOKEN', - )]) { - // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests - result = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'", - label: 'Check if CI should run slow tests', - ) - } - return result == 0 - } - - def cancel_previous_build() { - // cancel previous build if it is not on main. - if (env.BRANCH_NAME != 'main') { - def buildNumber = env.BUILD_NUMBER as int - // Milestone API allows us to cancel previous build - // with the same milestone number - if (buildNumber > 1) milestone(buildNumber - 1) - milestone(buildNumber) - } - } - - def is_last_build() { - // check whether it is last build - try { - return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number - } catch (Throwable ex) { - echo 'Error during check is_last_build ' + ex.toString() - return false - } - } - - def checkout_trusted_files() { - // trust everything from branch builds - if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { - return; - } - - // trust peoople listed in CONTRIBUTING.md - grep_code = sh( - returnStatus: true, - script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'", - label: 'Check if change is from a contributor', - ) - - if (grep_code == 1) { - // Any scripts that run on the bare host and not inside a Docker container - // (especially those that access secrets) should be checked out here so - // only trusted versions are used in CI - sh( - script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.", - label: 'Check out trusted files', - ) - } - } - - def should_skip_ci(pr_number) { - if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { - // never skip CI on build sourced from a branch - return false - } - glob_skip_ci_code = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_skip_ci_globs.py", - label: 'Check if CI should be skipped due to changed files', - ) - if (glob_skip_ci_code == 0) { - return true - } - withCredentials([string( - credentialsId: 'tvm-bot-jenkins-reader', - variable: 'GITHUB_TOKEN', - )]) { - // Exit code of 1 means run full CI (or the script had an error, so run - // full CI just in case). Exit code of 0 means skip CI. - git_skip_ci_code = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'", - label: 'Check if CI should be skipped', - ) - } - return git_skip_ci_code == 0 - } - - def prepare(node_type) { - stage('Prepare') { - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") { - init_git() - - if (env.DETERMINE_DOCKER_IMAGES == 'yes') { - sh( - script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cpu ci_gpu ci_wasm ", - label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images', - ) - // Pull image names from the results of should_rebuild_docker.py - ci_arm = sh( - script: "cat .docker-image-names/ci_arm", - label: "Find docker image name for ci_arm", - returnStdout: true, - ).trim() - ci_cpu = sh( - script: "cat .docker-image-names/ci_cpu", - label: "Find docker image name for ci_cpu", - returnStdout: true, - ).trim() - ci_gpu = sh( - script: "cat .docker-image-names/ci_gpu", - label: "Find docker image name for ci_gpu", - returnStdout: true, - ).trim() - ci_wasm = sh( - script: "cat .docker-image-names/ci_wasm", - label: "Find docker image name for ci_wasm", - returnStdout: true, - ).trim() - } - - ci_arm = params.ci_arm_param ?: ci_arm - ci_cpu = params.ci_cpu_param ?: ci_cpu - ci_gpu = params.ci_gpu_param ?: ci_gpu - ci_wasm = params.ci_wasm_param ?: ci_wasm - - sh (script: """ - echo "Docker images being used in this build:" - echo " ci_arm = ${ci_arm}" - echo " ci_cpu = ${ci_cpu}" - echo " ci_gpu = ${ci_gpu}" - echo " ci_wasm = ${ci_wasm}" - """, label: 'Docker image names') - - is_docs_only_build = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_change_docs.sh", - label: 'Check for docs only changes', - ) - skip_ci = should_skip_ci(env.CHANGE_ID) - skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) - rebuild_docker_images = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_change_docker.sh", - label: 'Check for any docker changes', - ) - - if (skip_ci) { - // Don't rebuild when skipping CI - rebuild_docker_images = false - } - } - } - } - } - def python_unittest(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", - label: 'Run Python unit tests', - ) - } - - // Keep compiler caching and executor limits identical for wheels and the - // separate runtime-only compile check. Build commands remain at their callers. - def build_environment() { - return ''' - set -eux - export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} )) - if [ "$CMAKE_BUILD_PARALLEL_LEVEL" -lt 1 ]; then export CMAKE_BUILD_PARALLEL_LEVEL=1; fi - if command -v sccache >/dev/null; then - export CC=/opt/sccache/cc CXX=/opt/sccache/c++ - if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then - export SCCACHE_BUCKET=tvm-sccache-prod SCCACHE_REGION=us-west-2 - fi - sccache --start-server || true - sccache --show-stats - trap "sccache --show-stats" EXIT - fi - ''' - } - - def wheel_build(image, cpp_tests = false) { - def commands = build_environment() + ''' - rm -rf build/wheels - uv build --wheel --python python3 --out-dir build/wheels ./3rdparty/tvm-ffi - uv build --wheel --python python3 --out-dir build/wheels . \ - -Ccmake.build-type=RelWithDebInfo -Cbuild.verbose="${VERBOSE:-true}" - '''.stripIndent().trim() - if (cpp_tests) { - commands += " -Ccmake.define.USE_GTEST=ON -Cbuild.targets=all\\;cpptest" - // cpptest links the same native tree that scikit-build just packaged. - commands += "\n./tests/scripts/task_cpp_unittest.sh build" - } - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${image} bash -c '${commands}'", - label: 'Build wheel artifacts' + (cpp_tests ? ' and run C++ tests' : ''), - ) - } - - cancel_previous_build() + // readTrusted pins trusted revisions, but rejects helper edits in untrusted PRs. + // In that case use the PR target, pinning the same commit for all source merges. + def helper + String target_revision = null try { - prepare('CPU-SMALL-SPOT') - } catch(Exception ex) { - prepare('CPU-SMALL') - } - def run_build(node_type) { - if (!skip_ci) { - echo 'Begin running node_type ' + node_type - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") { - init_git() - docker_init(ci_gpu) - timeout(time: max_time, unit: 'MINUTES') { - - withEnv([ - 'PLATFORM=gpu', - ], { - sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" - wheel_build("${ci_gpu} --no-gpu") - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu --bundle wheels --bundle tvm_runtime", - label: 'Upload artifacts to S3', - ) - - - // compiler test - sh "rm -rf build" - sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build" - def runtime_commands = build_environment() + ''' - cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo - cmake --build build --target tvm_runtime --verbose - ''' - sh "${docker_run} --env CI_NUM_EXECUTORS ${ci_gpu} --no-gpu bash -c '${runtime_commands}'" - }) - } + helper = readTrusted('ci/jenkins/generated/common.groovy') + } catch (hudson.AbortException | java.io.FileNotFoundException ex) { + if (!env.CHANGE_TARGET) throw ex + echo 'Loading shared pipeline from the pinned PR target: ' + ex.toString() + def from_target = { label -> + node(label) { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/pipeline") { + // No PR-owned executable or credentials run during this bootstrap. + deleteDir() + retry(5) { checkout scm } + // Quote the branch as data and fetch only from the canonical repository. + def target = env.CHANGE_TARGET.replace("'", "'\"'\"'") + sh(script: "git fetch https://github.com/apache/tvm.git 'refs/heads/${target}'", + label: 'Fetch trusted pipeline target') + def revision = sh(script: 'git rev-parse FETCH_HEAD', returnStdout: true).trim() + def source = sh(script: "git show ${revision}:ci/jenkins/generated/common.groovy", + returnStdout: true) + return [revision, source] } } - echo 'End running node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('BUILD: GPU') - } - } - def build() { - stage('Build') { - try { - run_build('CPU-SPOT') - } catch (hudson.AbortException abortEx) { - echo "Received normal AbortException, exit now. Details:" + abortEx.toString() - throw abortEx - } catch (Throwable ex) { - echo 'Exception during SPOT run ' + ex.toString() - if (is_last_build()) { - // retry if we are currently at last build - // mark the current stage as success - // and try again via on demand node - echo 'Retry on-demand given it is last build' - currentBuild.result = 'SUCCESS' - run_build('CPU') - } else { - echo 'Exit since it is not last build' - throw ex - } - } - } - } - build() - - - def run_unittest_GPU(node_type) { - echo 'Begin running on node_type ' + node_type - if (!skip_ci && is_docs_only_build != 1) { - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { - // NOTE: if exception happens, it will be caught outside - init_git() - docker_init(ci_gpu) - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=gpu', - 'TEST_STEP_NAME=unittest: GPU', - "SKIP_SLOW_TESTS=${skip_slow_tests}"], { - sh( - script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu", - label: 'Download artifacts from S3', - ) - - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh", - label: 'Run Java unit tests', - ) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", - label: 'Run Python GPU unit tests', - ) - }) - } - } - } - echo 'End running on node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('unittest: GPU') - } - } - - - - def run_docs_GPU(node_type) { - echo 'Begin running on node_type ' + node_type - if (!skip_ci) { - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/docs-python-gpu") { - // NOTE: if exception happens, it will be caught outside - init_git() - docker_init(ci_gpu) - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=gpu', - 'TEST_STEP_NAME=docs: GPU', - "SKIP_SLOW_TESTS=${skip_slow_tests}"], { - sh( - script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu", - label: 'Download artifacts from S3', - ) - - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh", - label: 'Build docs', - ) - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/docs --items docs.tgz", - label: 'Upload artifacts to S3', - ) - - sh( - script: "aws s3 cp --no-progress _docs s3://${s3_bucket}/${s3_prefix}/docs --recursive", - label: 'Upload docs to S3', - ) - }) - } - } - } - echo 'End running on node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('docs: GPU') } + def pinned + try { + pinned = from_target('CPU-SMALL-SPOT') + } catch (Exception failure) { + pinned = from_target('CPU-SMALL') + } + target_revision = pinned[0] + helper = pinned[1] + } + // Evaluation is outside the read/fetch fallback: script errors must propagate. + ci = evaluate(helper) + if (target_revision != null) ci.upstream_revision = target_revision + ci.start() + + ci.build('CPU', 'BUILD: GPU', 'tvm/build-gpu', images.ci_gpu, 'gpu') { + sh "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" - ci.cmake_build("${images.ci_gpu} --no-gpu", 'build') - ci.upload('gpu', '--bundle tvm_lib --bundle tvm_lib_gpu_extra') ++ ci.wheel_build("--no-gpu ${images.ci_gpu}") ++ ci.upload('gpu', '--bundle wheels --bundle tvm_runtime') + + // Check the compiler-only GPU configuration after publishing the test build. + sh 'rm -rf build' + sh "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build" - sh "${docker_run} --env CI_NUM_EXECUTORS ${images.ci_gpu} --no-gpu ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --cmake-target tvm_runtime --build-dir build" ++ def commands = ci.build_environment() + ''' ++ cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo ++ cmake --build build --target tvm_runtime ++ ''' ++ sh(ci.build_docker() + "--no-gpu ${images.ci_gpu} bash -c '${commands}'") } - - - def test() { - stage('Test') { - environment { - SKIP_SLOW_TESTS = "${skip_slow_tests}" - } - parallel( + stage('Test') { + parallel( 'unittest: GPU': { - try { - run_unittest_GPU('GPU-SPOT') - } catch (Throwable ex) { - echo 'Exception during SPOT run ' + ex.toString() - if (is_last_build()) { - // retry if at last build - // mark the current stage as success - // and try again via on demand node - echo 'Retry on-demand given it is last build' - currentBuild.result = 'SUCCESS' - run_unittest_GPU('GPU') - } else { - echo 'Exit since it is not last build' - throw ex - } + ci.test('GPU', 'unittest: GPU', 'tvm/ut-python-gpu', images.ci_gpu, 'gpu') { + ci.download('gpu') + sh( + script: "${docker_run} ${images.ci_gpu} ./tests/scripts/task_java_unittest.sh", + label: 'Run Java unit tests', + ) + sh( + script: "${docker_run} ${images.ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", + label: 'Run Python GPU unit tests', + ) } }, 'docs: GPU': { diff --cc ci/jenkins/generated/wasm_jenkinsfile.groovy index cd9861e7f6,1887926335..3b8ad52929 --- a/ci/jenkins/generated/wasm_jenkinsfile.groovy +++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy @@@ -14,498 -14,57 +14,55 @@@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - // -*- mode: groovy -*- - // Licensed to the Apache Software Foundation (ASF) under one - // or more contributor license agreements. See the NOTICE file - // distributed with this work for additional information - // regarding copyright ownership. The ASF licenses this file - // to you under the Apache License, Version 2.0 (the - // "License"); you may not use this file except in compliance - // with the License. You may obtain a copy of the License at - // - // http://www.apache.org/licenses/LICENSE-2.0 - // - // Unless required by applicable law or agreed to in writing, - // software distributed under the License is distributed on an - // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - // KIND, either express or implied. See the License for the - // specific language governing permissions and limitations - // under the License. - - // Jenkins pipeline - // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ - - // Docker env used for testing - // Different image may have different version tag - // because some of them are more stable than anoter. - // - // Docker images are maintained by PMC, cached in dockerhub - // and remains relatively stable over the time. - // Flow for upgrading docker env(need commiter) - // - // - Send PR to upgrade build script in the repo - // - Build the new docker image - // - Tag the docker image with a new version and push to a binary cache. - // - Update the version in the Jenkinsfile, send a PR - // - Fix any issues wrt to the new image version in the PR - // - Merge the PR and now we are in new version - // - Tag the new version as the lates - // - Periodically cleanup the old versions on local workers - // - - // ============================= IMPORTANT NOTE ============================= - // This file is generated by 'jenkins/generate.py'. Do not edit this file directly! - // Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with - // 'python3 jenkins/generate.py' - // Note: This timestamp is here to ensure that updates to the Jenkinsfile are - // always rebased on main before merging: - // Generated at 2026-10-14T21:26:39.087178 - - import org.jenkinsci.plugins.pipeline.modeldefinition.Utils - // These are set at runtime from data in ci/jenkins/docker-images.yml, update - // image tags in that file - ci_gpu = '' - ci_cpu = '' - ci_minimal = '' - ci_wasm = '' - ci_cortexm = '' - ci_arm = '' - ci_riscv = '' - - // Parameters to allow overriding (in Jenkins UI), the images - // to be used by a given build. When provided, they take precedence - // over default values above. - properties([ - parameters([ - string(name: 'ci_arm_param', defaultValue: ''), - string(name: 'ci_cpu_param', defaultValue: ''), - string(name: 'ci_gpu_param', defaultValue: ''), - string(name: 'ci_wasm_param', defaultValue: ''), - ]) - ]) - - // Placeholders for newly built Docker image names (if rebuild_docker_images - // is used) - built_ci_arm = null; - built_ci_cpu = null; - built_ci_gpu = null; - built_ci_wasm = null; - - // Global variable assigned during Sanity Check that holds the sha1 which should be - // merged into the PR in all branches. - upstream_revision = null - - // command to start a docker container - docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' - docker_build = 'docker/build.sh' - // timeout in minutes - max_time = 180 - rebuild_docker_images = false - - s3_bucket = 'tvm-jenkins-artifacts-prod' - s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" - - // Jenkins script root directory - jenkins_scripts_root = "ci/scripts/jenkins" - - - // General note: Jenkins has limits on the size of a method (or top level code) - // that are pretty strict, so most usage of groovy methods in these templates - // are purely to satisfy the JVM - def per_exec_ws(folder) { - return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder - } - - // initialize source codes - def init_git() { - retry(5) { - checkout scm - } - - // Add more info about job node - sh ( - script: './tests/scripts/task_show_node_info.sh', - label: 'Show executor node info', - ) - - // Determine merge commit to use for all stages - if (env.CHANGE_TARGET) { - // This is a PR build, so merge with the latest of the PR's target branch - // (e.g. main or a release branch like v0.25.0). - merge_with_target() - } else { - // This is a branch build (main or a release branch). Only set - // upstream_revision to HEAD and skip merging to avoid a race with another - // commit merged to the branch. - update_upstream_revision("HEAD") - } - - sh( - script: """ - set -eux - . ${jenkins_scripts_root}/retry.sh - retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0 - """, - label: 'Update git submodules', - ) - checkout_trusted_files() - } - - def update_upstream_revision(git_ref) { - if (upstream_revision == null) { - upstream_revision = sh( - script: "git log -1 ${git_ref} --format=\'%H\'", - label: 'Determine upstream revision', - returnStdout: true, - ).trim() - } - } - - def merge_with_target() { - def target = env.CHANGE_TARGET - sh ( - script: "git fetch origin ${target}", - label: "Fetch target branch ${target}", - ) - update_upstream_revision("FETCH_HEAD") - sh ( - script: "git -c user.name=TVM-Jenkins -c [email protected] merge ${upstream_revision}", - label: "Merge to origin/${target}" - ) - } - - def docker_init(image) { - // Clear out all Docker images that aren't going to be used - sh( - script: """ - set -eux - docker image ls --all - IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}') - - echo -e "Found images:\\n\$IMAGES" - echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; } - - docker image ls --all - """, - label: 'Clean old Docker images', - ) - - if (image.contains("amazonaws.com")) { - // If this string is in the image name it's from ECR and needs to be pulled - // with the right credentials - ecr_pull(image) - } else { - sh( - script: """ - set -eux - . ${jenkins_scripts_root}/retry.sh - retry 5 docker pull ${image} - """, - label: 'Pull docker image', - ) - } - } - - def ecr_pull(full_name) { - aws_account_id = sh( - returnStdout: true, - script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', - label: 'Get AWS ID' - ).trim() - - try { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2', - "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { - sh( - script: ''' - set -eux - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO - ''', - label: 'Log in to ECR' - ) - sh( - script: """ - set -eux - . ${jenkins_scripts_root}/retry.sh - retry 5 docker pull ${full_name} - """, - label: 'Pull image from ECR' - ) - } - } finally { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2', - "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { - sh( - script: 'docker logout $AWS_ECR_REPO', - label: 'Clean up login credentials' - ) + // readTrusted pins trusted revisions, but rejects helper edits in untrusted PRs. + // In that case use the PR target, pinning the same commit for all source merges. + def helper + String target_revision = null + try { + helper = readTrusted('ci/jenkins/generated/common.groovy') + } catch (hudson.AbortException | java.io.FileNotFoundException ex) { + if (!env.CHANGE_TARGET) throw ex + echo 'Loading shared pipeline from the pinned PR target: ' + ex.toString() + def from_target = { label -> + node(label) { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/pipeline") { + // No PR-owned executable or credentials run during this bootstrap. + deleteDir() + retry(5) { checkout scm } + // Quote the branch as data and fetch only from the canonical repository. + def target = env.CHANGE_TARGET.replace("'", "'\"'\"'") + sh(script: "git fetch https://github.com/apache/tvm.git 'refs/heads/${target}'", + label: 'Fetch trusted pipeline target') + def revision = sh(script: 'git rev-parse FETCH_HEAD', returnStdout: true).trim() + def source = sh(script: "git show ${revision}:ci/jenkins/generated/common.groovy", + returnStdout: true) + return [revision, source] + } } } - } - - def should_skip_slow_tests(pr_number) { - withCredentials([string( - credentialsId: 'tvm-bot-jenkins-reader', - variable: 'GITHUB_TOKEN', - )]) { - // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests - result = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'", - label: 'Check if CI should run slow tests', - ) - } - return result == 0 - } - - def cancel_previous_build() { - // cancel previous build if it is not on main. - if (env.BRANCH_NAME != 'main') { - def buildNumber = env.BUILD_NUMBER as int - // Milestone API allows us to cancel previous build - // with the same milestone number - if (buildNumber > 1) milestone(buildNumber - 1) - milestone(buildNumber) - } - } - - def is_last_build() { - // check whether it is last build + def pinned try { - return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number - } catch (Throwable ex) { - echo 'Error during check is_last_build ' + ex.toString() - return false + pinned = from_target('CPU-SMALL-SPOT') + } catch (Exception failure) { + pinned = from_target('CPU-SMALL') } + target_revision = pinned[0] + helper = pinned[1] } + // Evaluation is outside the read/fetch fallback: script errors must propagate. + ci = evaluate(helper) + if (target_revision != null) ci.upstream_revision = target_revision + ci.start() - def checkout_trusted_files() { - // trust everything from branch builds - if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { - return; - } - - // trust peoople listed in CONTRIBUTING.md - grep_code = sh( - returnStatus: true, - script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'", - label: 'Check if change is from a contributor', - ) - - if (grep_code == 1) { - // Any scripts that run on the bare host and not inside a Docker container - // (especially those that access secrets) should be checked out here so - // only trusted versions are used in CI - sh( - script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.", - label: 'Check out trusted files', - ) - } - } - - def should_skip_ci(pr_number) { - if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) { - // never skip CI on build sourced from a branch - return false - } - glob_skip_ci_code = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_skip_ci_globs.py", - label: 'Check if CI should be skipped due to changed files', - ) - if (glob_skip_ci_code == 0) { - return true - } - withCredentials([string( - credentialsId: 'tvm-bot-jenkins-reader', - variable: 'GITHUB_TOKEN', - )]) { - // Exit code of 1 means run full CI (or the script had an error, so run - // full CI just in case). Exit code of 0 means skip CI. - git_skip_ci_code = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'", - label: 'Check if CI should be skipped', - ) - } - return git_skip_ci_code == 0 - } - - def prepare(node_type) { - stage('Prepare') { - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") { - init_git() - - if (env.DETERMINE_DOCKER_IMAGES == 'yes') { - sh( - script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cpu ci_gpu ci_wasm ", - label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images', - ) - // Pull image names from the results of should_rebuild_docker.py - ci_arm = sh( - script: "cat .docker-image-names/ci_arm", - label: "Find docker image name for ci_arm", - returnStdout: true, - ).trim() - ci_cpu = sh( - script: "cat .docker-image-names/ci_cpu", - label: "Find docker image name for ci_cpu", - returnStdout: true, - ).trim() - ci_gpu = sh( - script: "cat .docker-image-names/ci_gpu", - label: "Find docker image name for ci_gpu", - returnStdout: true, - ).trim() - ci_wasm = sh( - script: "cat .docker-image-names/ci_wasm", - label: "Find docker image name for ci_wasm", - returnStdout: true, - ).trim() - } - - ci_arm = params.ci_arm_param ?: ci_arm - ci_cpu = params.ci_cpu_param ?: ci_cpu - ci_gpu = params.ci_gpu_param ?: ci_gpu - ci_wasm = params.ci_wasm_param ?: ci_wasm - - sh (script: """ - echo "Docker images being used in this build:" - echo " ci_arm = ${ci_arm}" - echo " ci_cpu = ${ci_cpu}" - echo " ci_gpu = ${ci_gpu}" - echo " ci_wasm = ${ci_wasm}" - """, label: 'Docker image names') - - is_docs_only_build = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_change_docs.sh", - label: 'Check for docs only changes', - ) - skip_ci = should_skip_ci(env.CHANGE_ID) - skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) - rebuild_docker_images = sh ( - returnStatus: true, - script: "./${jenkins_scripts_root}/git_change_docker.sh", - label: 'Check for any docker changes', - ) - - if (skip_ci) { - // Don't rebuild when skipping CI - rebuild_docker_images = false - } - } - } - } - } - def python_unittest(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", - label: 'Run Python unit tests', + ci.build('CPU', 'BUILD: WASM', 'tvm/build-wasm', images.ci_wasm, 'wasm') { + sh( + script: "${docker_run} ${images.ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", + label: 'Create WASM CMake config', ) - } - - // Keep compiler caching and executor limits identical for wheels and the - // separate runtime-only compile check. Build commands remain at their callers. - def build_environment() { - return ''' - set -eux - export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} )) - if [ "$CMAKE_BUILD_PARALLEL_LEVEL" -lt 1 ]; then export CMAKE_BUILD_PARALLEL_LEVEL=1; fi - if command -v sccache >/dev/null; then - export CC=/opt/sccache/cc CXX=/opt/sccache/c++ - if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then - export SCCACHE_BUCKET=tvm-sccache-prod SCCACHE_REGION=us-west-2 - fi - sccache --start-server || true - sccache --show-stats - trap "sccache --show-stats" EXIT - fi - ''' - } - - def wheel_build(image, cpp_tests = false) { - def commands = build_environment() + ''' - rm -rf build/wheels - uv build --wheel --python python3 --out-dir build/wheels ./3rdparty/tvm-ffi - uv build --wheel --python python3 --out-dir build/wheels . \ - -Ccmake.build-type=RelWithDebInfo -Cbuild.verbose="${VERBOSE:-true}" - '''.stripIndent().trim() - if (cpp_tests) { - commands += " -Ccmake.define.USE_GTEST=ON -Cbuild.targets=all\\;cpptest" - // cpptest links the same native tree that scikit-build just packaged. - commands += "\n./tests/scripts/task_cpp_unittest.sh build" - } - sh ( - script: "${docker_run} --env CI_NUM_EXECUTORS ${image} bash -c '${commands}'", - label: 'Build wheel artifacts' + (cpp_tests ? ' and run C++ tests' : ''), - ci.cmake_build(images.ci_wasm, 'build') - ci.make_cpp_tests(images.ci_wasm, 'build') - ci.cpp_unittest(images.ci_wasm) ++ ci.wheel_build(images.ci_wasm, true) + sh( + script: "${docker_run} ${images.ci_wasm} ./tests/scripts/task_web_wasm.sh", + label: 'Run WASM lint and tests', ) } - - cancel_previous_build() - - try { - prepare('CPU-SMALL-SPOT') - } catch(Exception ex) { - prepare('CPU-SMALL') - } - def run_build(node_type) { - if (!skip_ci && is_docs_only_build != 1) { - echo 'Begin running node_type ' + node_type - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") { - init_git() - docker_init(ci_wasm) - timeout(time: max_time, unit: 'MINUTES') { - - withEnv([ - 'PLATFORM=wasm', - ], { - sh ( - script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", - label: 'Create WASM CMake config', - ) - wheel_build(ci_wasm, true) - sh ( - script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh", - label: 'Run WASM lint and tests', - ) - }) - } - } - } - echo 'End running node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('BUILD: WASM') - } - } - def build() { - stage('Build') { - try { - run_build('CPU-SPOT') - } catch (hudson.AbortException abortEx) { - echo "Received normal AbortException, exit now. Details:" + abortEx.toString() - throw abortEx - } catch (Throwable ex) { - echo 'Exception during SPOT run ' + ex.toString() - if (is_last_build()) { - // retry if we are currently at last build - // mark the current stage as success - // and try again via on demand node - echo 'Retry on-demand given it is last build' - currentBuild.result = 'SUCCESS' - run_build('CPU') - } else { - echo 'Exit since it is not last build' - throw ex - } - } - } - } - build() diff --cc tests/python/ci/test_jenkins_helpers.py index 0000000000,b9d26a78a0..23c0e42383 mode 000000,100644..100644 --- a/tests/python/ci/test_jenkins_helpers.py +++ b/tests/python/ci/test_jenkins_helpers.py @@@ -1,0 -1,134 +1,80 @@@ + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, + # software distributed under the License is distributed on an + # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + # KIND, either express or implied. See the License for the + # specific language governing permissions and limitations + # under the License. + """Host-side Jenkins commands preserve retries, paths and build configuration.""" + + import importlib.util + import json + import os -import shutil + import subprocess -import sys + + import pytest + -from .test_utils import JENKINS_SCRIPT_ROOT, REPO_ROOT ++from .test_utils import JENKINS_SCRIPT_ROOT + + + def load_s3(): + spec = importlib.util.spec_from_file_location("jenkins_s3", JENKINS_SCRIPT_ROOT / "s3.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + + def executable(path, body): + path.write_text("#!/usr/bin/env python3\n" + body) + path.chmod(0o755) + + + @pytest.mark.parametrize("failures", [2, 3]) + def test_s3_transfer_retries(tmp_path, monkeypatch, failures): + # Run the real retry.sh with fake AWS and sleep, without network or delays. + executable( + tmp_path / "aws", + """ + import json, os, pathlib, sys + log = pathlib.Path(os.environ["AWS_LOG"]) + with log.open("a") as out: + out.write(json.dumps(sys.argv[1:]) + "\\n") + attempt = len(log.read_text().splitlines()) + print("download: s3://bucket/file to file name") + sys.exit(1 if attempt <= int(os.environ["FAILURES"]) else 0) + """, + ) + executable( + tmp_path / "sleep", + """ + import os, pathlib, sys + with pathlib.Path(os.environ["SLEEP_LOG"]).open("a") as out: + out.write(sys.argv[1] + "\\n") + """, + ) + monkeypatch.setenv("PATH", f"{tmp_path}:{os.environ['PATH']}") + monkeypatch.setenv("FAILURES", str(failures)) + monkeypatch.setenv("SLEEP_LOG", str(tmp_path / "sleeps")) + s3 = load_s3() + for transfer in range(2): + log = tmp_path / f"aws-{transfer}" + monkeypatch.setenv("AWS_LOG", str(log)) + if failures == 3: + with pytest.raises(subprocess.CalledProcessError): + s3.s3("s3://bucket/file", "file name", False) + else: + assert s3.parse_output_files(s3.s3("s3://bucket/file", "file name", False)) + commands = [json.loads(line) for line in log.read_text().splitlines()] + assert commands == [["s3", "cp", "--no-progress", "s3://bucket/file", "file name"]] * 3 + delays = (tmp_path / "sleeps").read_text().splitlines() + assert len(delays) == 4 + assert all(30 <= int(delay) <= 200 for delay in delays) - - -def test_build_environment_and_target(tmp_path): - # Run the entry point in a minimal checkout, capturing the tool boundary. - script = tmp_path / "tests/scripts/task_build.py" - script.parent.mkdir(parents=True) - shutil.copy(REPO_ROOT / "tests/scripts/task_build.py", script) - (tmp_path / "build").mkdir() - executable( - tmp_path / "cmake", - """ -import json, os, pathlib, sys -with pathlib.Path(os.environ["COMMAND_LOG"]).open("a") as out: - entry = [sys.argv[1:], os.getcwd(), os.environ["SCCACHE_BUCKET"], os.environ["CC"]] - out.write(json.dumps(entry) + "\\n") -""", - ) - executable(tmp_path / "sccache", "pass\n") - log = tmp_path / "commands" - env = dict( - os.environ, - PATH=f"{tmp_path}:{os.environ['PATH']}", - COMMAND_LOG=str(log), - AWS_ACCESS_KEY_ID="test", - CI_NUM_EXECUTORS="2", - VERBOSE="false", - ) - subprocess.run( - [ - sys.executable, - script, - "--debug", - "--cmake-target", - "cpptest", - "--sccache-bucket", - "test-cache", - ], - cwd=tmp_path, - env=env, - check=True, - ) - configure, build = [json.loads(line) for line in log.read_text().splitlines()] - assert configure == [ - ["-GNinja", "-DCMAKE_BUILD_TYPE=Debug", ".."], - str(tmp_path / "build"), - "test-cache", - "/opt/sccache/cc", - ] - assert build[0][:4] == ["--build", ".", "--", "cpptest"] - assert build[0][4].startswith("-j") and int(build[0][4][2:]) >= 1 - assert len(build[0]) == 5 - assert build[1:] == configure[1:] diff --cc tests/python/ci/test_wheel_artifacts.py index 9eaa8613bc,0000000000..e764f3b27e mode 100644,000000..100644 --- a/tests/python/ci/test_wheel_artifacts.py +++ b/tests/python/ci/test_wheel_artifacts.py @@@ -1,167 -1,0 +1,189 @@@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Wheel artifact transport and local CI command contracts, without a TVM import.""" + +import importlib.util +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[3] + + +def load_local_ci(): + spec = importlib.util.spec_from_file_location("local_ci", ROOT / "tests/scripts/ci.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + [email protected]("platform", ["cpu", "gpu", "arm", "wasm"]) +def test_local_wheel_build(monkeypatch, platform): + ci = load_local_ci() + calls = [] + monkeypatch.setattr(ci, "docker", lambda **kwargs: calls.append(kwargs)) + command, _, _ = ci.generate_command(platform, {}, "wheel build") + command(cpp=True) + scripts = calls[0]["scripts"] + wheel = next(script for script in scripts if "-Cbuild-dir=" in script) + assert "-Ccmake.build-type=RelWithDebInfo" in wheel + assert "all;cpptest" in wheel - assert any("./3rdparty/tvm-ffi" in script for script in scripts) - assert any("task_python_install_wheels.sh" in script for script in scripts) == ( - platform != "wasm" - ) ++ assert not any("./3rdparty/tvm-ffi" in script for script in scripts) ++ assert "--no-build-isolation --offline" in wheel ++ assert any("task_python_install_wheels.sh" in script for script in scripts) + calls.clear() + command(skip_build=True, tests=["tests/python/example.py"]) + assert not any("uv build" in script for script in calls[0]["scripts"]) + + [email protected]("succeed", [True, False]) +def test_wheel_directory_upload_retries_each_transfer(tmp_path, succeed): + # Exercise the existing S3 CLI/retry boundary with a directory bundle. + for relative in [ - "ci/scripts/jenkins/s3.py", - "ci/scripts/jenkins/cmd_utils.py", - "ci/scripts/jenkins/retry.sh", ++ "ci/jenkins/scripts/s3.py", ++ "ci/jenkins/scripts/retry.sh", + "ci/jenkins/data.py", + ]: + target = tmp_path / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(ROOT / relative, target) + wheels = tmp_path / "build/wheels" + wheels.mkdir(parents=True) + (wheels / "apache_tvm.whl").write_text("tvm") - (wheels / "apache_tvm_ffi.whl").write_text("ffi") + binaries = tmp_path / "bin" + binaries.mkdir() + aws = binaries / "aws" + aws.write_text( + f"#!{sys.executable}\n" + "import json, pathlib, sys\n" + "path = pathlib.Path('attempts.json')\n" + "calls = json.loads(path.read_text()) if path.exists() else []\n" + "calls.append(sys.argv[1:])\n" + "path.write_text(json.dumps(calls))\n" + f"sys.exit(0 if {succeed!r} and len(calls) == 3 else 1)\n" + ) + aws.chmod(0o755) + sleep = binaries / "sleep" + sleep.write_text('#!/bin/sh\necho "$1" >> delays.txt\n') + sleep.chmod(0o755) + result = subprocess.run( + [ + sys.executable, - "ci/scripts/jenkins/s3.py", ++ "ci/jenkins/scripts/s3.py", + "--action", + "upload", + "--bucket", + "test", + "--prefix", + "cpu", + "--bundle", + "wheels", + ], + cwd=tmp_path, + env={**os.environ, "PATH": f"{binaries}:{os.environ['PATH']}"}, + capture_output=True, + text=True, + ) + assert (result.returncode == 0) == succeed, result.stdout + result.stderr + attempts = json.loads((tmp_path / "attempts.json").read_text()) + assert len(attempts) == 3 + assert all( + call[-3:] == ["--recursive", "build/wheels", "s3://test/cpu/build/wheels"] + for call in attempts + ) + delays = [int(line) for line in (tmp_path / "delays.txt").read_text().splitlines()] + assert len(delays) == 2 and all(30 <= delay <= 200 for delay in delays) + + [email protected]("fail", [False, True]) +def test_local_interactive_keeps_cache_stats(tmp_path, monkeypatch, fail): + ci = load_local_ci() + monkeypatch.setattr(ci, "check_docker", lambda: None) + monkeypatch.setattr(ci, "cmd", lambda *args, **kwargs: None) + monkeypatch.setattr(ci, "SCRIPT_DIR", tmp_path / "scripts") + monkeypatch.setattr(ci, "REPO_ROOT", tmp_path) + monkeypatch.delenv("USE_SCCACHE", raising=False) + monkeypatch.setenv("DEBUG", "1") + binaries = tmp_path / "bin" + binaries.mkdir() + for name in ["sccache", "bash"]: + binary = binaries / name + binary.write_text(f'#!/bin/sh\necho "{name} $*" >> "$CALL_LOG"\n') + binary.chmod(0o755) + ci.docker("interactive", "ci_cpu", ["false" if fail else "true"], {}, True) + result = subprocess.run( + [shutil.which("bash"), str(ci.SCRIPT_DIR / "interactive.sh")], + env={ + **os.environ, + "PATH": f"{binaries}:{os.environ['PATH']}", + "CALL_LOG": str(tmp_path / "calls"), + }, + capture_output=True, + text=True, + ) + assert (result.returncode != 0) == fail + calls = (tmp_path / "calls").read_text().splitlines() + assert calls == [ + "sccache --start-server", + "sccache --show-stats", + "sccache --show-stats", + "bash ", + ] + + [email protected]("wheel_docs", [False, True]) +def test_docs_import_path(tmp_path, monkeypatch, wheel_docs): + # Run the real Sphinx path bootstrap without requiring Sphinx extensions. + # A stale --target=python FFI package must not precede Jenkins' wheel install. + monkeypatch.chdir(ROOT / "docs") + source_python = str(ROOT / "python") + monkeypatch.setattr(sys, "path", [path for path in sys.path if path != source_python]) + monkeypatch.setenv("TVM_DOCS_USE_WHEEL", "1" if wheel_docs else "0") + config = ROOT / "docs/conf.py" + bootstrap = config.read_text().split("# -- General configuration")[0] + exec(compile(bootstrap, str(config), "exec"), {"__file__": str(config)}) + assert (source_python in sys.path) != wheel_docs ++ ++ [email protected]( ++ "host_limit,caller_limit,expected", [(None, None, "2"), ("7", None, "7"), ("7", "9", "9")] ++) ++def test_local_parallel_limit_crosses_docker_boundary( ++ tmp_path, monkeypatch, host_limit, caller_limit, expected ++): ++ ci = load_local_ci() ++ calls = [] ++ monkeypatch.setattr(ci, "check_docker", lambda: None) ++ monkeypatch.setattr(ci, "cmd", lambda command: calls.append(command)) ++ monkeypatch.setattr(ci, "SCRIPT_DIR", tmp_path / "scripts") ++ monkeypatch.setattr(ci, "REPO_ROOT", tmp_path) ++ monkeypatch.setattr(ci, "NPROC", 8) ++ monkeypatch.setenv("CI_NUM_EXECUTORS", "3") ++ if host_limit is None: ++ monkeypatch.delenv("CMAKE_BUILD_PARALLEL_LEVEL", raising=False) ++ else: ++ monkeypatch.setenv("CMAKE_BUILD_PARALLEL_LEVEL", host_limit) ++ env = {} if caller_limit is None else {"CMAKE_BUILD_PARALLEL_LEVEL": caller_limit} ++ ci.docker("build", "ci_cpu", ["true"], env, False) ++ args = calls[0] ++ index = args.index(f"CMAKE_BUILD_PARALLEL_LEVEL={expected}") ++ assert args[index - 1] == "--env" diff --cc tests/scripts/ci.py index f242f110b5,fa027c150a..ec51674b05 --- a/tests/scripts/ci.py +++ b/tests/scripts/ci.py @@@ -177,12 -183,9 +177,14 @@@ def docker env["SCCACHE_CACHE_SIZE"] = os.getenv("SCCACHE_CACHE_SIZE", "50G") env["SCCACHE_SERVER_PORT"] = os.getenv("SCCACHE_SERVER_PORT", "4226") - # Match Jenkins' per-executor native build budget, including wheel builds. - scripts = [ - "export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(getconf _NPROCESSORS_ONLN) / " - "${CI_NUM_EXECUTORS:-1} ))", - 'if [ "$CMAKE_BUILD_PARALLEL_LEVEL" -lt 1 ]; then export CMAKE_BUILD_PARALLEL_LEVEL=1; fi', - ] + scripts ++ # Pass the bound through Docker; CMake and scikit-build honor it directly. ++ env.setdefault( ++ "CMAKE_BUILD_PARALLEL_LEVEL", ++ os.environ.get( ++ "CMAKE_BUILD_PARALLEL_LEVEL", ++ str(max(NPROC // int(os.environ.get("CI_NUM_EXECUTORS", "1")), 1)), ++ ), ++ ) env["PLATFORM"] = name docker_bash = REPO_ROOT / "docker" / "bash.sh" @@@ -400,22 -402,9 +402,20 @@@ def generate_command else: scripts = [ f"./tests/scripts/task_config_build_{name}.sh {build_dir}", - f"./tests/scripts/task_build.py --build-dir {build_dir}", + f"rm -rf {build_dir}/wheels", - f"uv build --wheel --python python3 --out-dir {build_dir}/wheels ./3rdparty/tvm-ffi", - f"uv build --wheel --python python3 --out-dir {build_dir}/wheels . " ++ f"uv build --wheel --no-build-isolation --offline --python python3 --out-dir {build_dir}/wheels . " + f"-Cbuild-dir={build_dir} -Ccmake.build-type=RelWithDebInfo " + f"-Cbuild.verbose={'true' if verbose else 'false'}" + + ( + " -Ccmake.define.USE_GTEST=ON '-Cbuild.targets=all;cpptest'" + if kwargs.get("cpp") + else "" + ), ] + # Direct --tests and interactive Python use the same installed artifacts. - if name != "wasm": - scripts.append("source ./tests/scripts/task_python_install_wheels.sh") ++ scripts.append("source ./tests/scripts/task_python_install_wheels.sh") + if post_build is not None: scripts += post_build @@@ -433,10 -422,10 +433,7 @@@ scripts.extend(script.format(build_dir=build_dir) for script in extra_scripts) docker_env = { - # The specialized Wasm source tests retain a native build path; - # the wheel installer clears it for Python wheel consumers. - # Need to specify the library path manually or else TVM can't - # determine which build directory to use (i.e. if there are - # multiple copies of libtvm_compiler.so / libtvm_runtime.so laying around) -- "TVM_LIBRARY_PATH": str(REPO_ROOT / get_build_dir(name)), + "TVM_WHEEL_DIR": str(REPO_ROOT / build_dir / "wheels"), "VERBOSE": "true" if verbose else "false", } diff --cc tests/scripts/task_python_docs.sh index 73d2662465,e45d439929..878996fa12 --- a/tests/scripts/task_python_docs.sh +++ b/tests/scripts/task_python_docs.sh @@@ -18,14 -18,7 +18,13 @@@ set -euxo pipefail -export PYTHONPATH="$(pwd)/python" +# Local docs retain source builds; Jenkins imports the exact build-node wheels. +if [ "${IS_LOCAL:-0}" != "1" ]; then + source "$(dirname "$0")/task_python_install_wheels.sh" + export TVM_DOCS_USE_WHEEL=1 +else - uv pip install -v --target=python ./3rdparty/tvm-ffi/ + export PYTHONPATH="$(pwd)/python" +fi # to avoid CI CPU thread throttling. export TVM_BIND_THREADS=0 @@@ -133,8 -129,11 +132,6 @@@ clean_file find . -type f -path "*.log" | xargs rm -f find . -type f -path "*.pyc" | xargs rm -f -# setup tvm-ffi into python folder -uv pip install -v --target=python ./3rdparty/tvm-ffi/ - --# LLVMModule JIT execution, needed by the tutorials this build runs. --uv pip install --no-deps apache-tvm-ffi-orcjit==0.1.2 cd docs diff --cc tests/scripts/task_python_install_wheels.sh index f54c57c9c5,0000000000..0138187eac mode 100644,000000..100644 --- a/tests/scripts/task_python_install_wheels.sh +++ b/tests/scripts/task_python_install_wheels.sh @@@ -1,54 -1,0 +1,72 @@@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Source this file so subsequent Python invocations use the wheel environment. +# Docker defaults PYTHONPATH to the checkout; never let it shadow the artifacts. +unset PYTHONPATH TVM_LIBRARY_PATH TVM_FFI_LIBRARY_PATH +export PYTHONNOUSERSITE=1 +TVM_WHEEL_DIR="$(realpath "${TVM_WHEEL_DIR:-$(pwd)/build/wheels}")" +# Preserve the image's active environment (often itself a venv), including when +# local ci.py sources this setup again for a named test suite. +export TVM_IMAGE_PYTHON="${TVM_IMAGE_PYTHON:-$(command -v python3)}" +export TVM_IMAGE_SITE_PACKAGES="${TVM_IMAGE_SITE_PACKAGES:-$("$TVM_IMAGE_PYTHON" -I -c 'import site; print("\n".join(site.getsitepackages()))')}" +export VIRTUAL_ENV="${TVM_WHEEL_DIR%/}/../wheel-venv" - uv venv --python "$TVM_IMAGE_PYTHON" --clear "$VIRTUAL_ENV" ++uv venv --offline --python "$TVM_IMAGE_PYTHON" --clear "$VIRTUAL_ENV" +# Append the image dependencies after our wheels without executing its .pth files: +# system-site-packages can install editable-import hooks ahead of the wheel. +TVM_WHEEL_SITE="$("$VIRTUAL_ENV/bin/python3" -c 'import sysconfig; print(sysconfig.get_path("purelib"))')" +printf '%s\n' "$TVM_IMAGE_SITE_PACKAGES" > "$TVM_WHEEL_SITE/image-dependencies.pth" +export PATH="$VIRTUAL_ENV/bin:$PATH" - uv pip install --python "$VIRTUAL_ENV/bin/python3" --no-deps --reinstall \ - "$TVM_WHEEL_DIR"/apache_tvm-*.whl "$TVM_WHEEL_DIR"/apache_tvm_ffi-*.whl ++uv pip install --python "$VIRTUAL_ENV/bin/python3" --no-deps --offline --reinstall \ ++ "$TVM_WHEEL_DIR"/apache_tvm-*.whl + +# Fail before pytest if a source/editable install or native override leaked in. +python3 - <<'PYTHON' - from pathlib import Path ++import os +import sys ++from importlib.metadata import requires, version ++from pathlib import Path ++from packaging.requirements import Requirement ++ ++# --no-deps leaves dependency ownership with the image. Check the wheel's own ++# requirements without invoking a resolver or accepting a stale hardcoded floor. ++for text in requires("apache-tvm") or []: ++ requirement = Requirement(text) ++ if requirement.marker and not requirement.marker.evaluate({"extra": ""}): ++ continue ++ installed = version(requirement.name) ++ assert requirement.specifier.contains(installed, prereleases=True), (text, installed) ++ +import tvm +import tvm_ffi + +prefix = Path(sys.prefix).resolve() - for module in (tvm, tvm_ffi): - path = Path(module.__file__).resolve() - assert path.is_relative_to(prefix), path - print(f"Installed {module.__name__}: {path}") - for name, library in {"tvm_ffi": tvm_ffi.LIB, **tvm.base._LOADED_LIBS}.items(): ++path = Path(tvm.__file__).resolve() ++assert path.is_relative_to(prefix), path ++print(f"Installed tvm: {path}") ++# FFI is supplied by the image, not rebuilt or transferred with TVM. ++image_sites = [Path(p).resolve() for p in os.environ["TVM_IMAGE_SITE_PACKAGES"].splitlines()] ++for path in (Path(tvm_ffi.__file__).resolve(), Path(tvm_ffi.LIB._name).resolve()): ++ assert any(path.is_relative_to(site) for site in image_sites), path ++ print(f"Image FFI: {path}") ++assert {"tvm_runtime", "tvm_compiler"} <= tvm.base._LOADED_LIBS.keys() ++for name, library in tvm.base._LOADED_LIBS.items(): + path = Path(library._name).resolve() + assert path.is_relative_to(prefix), (name, path) + print(f"Installed {name}: {path}") +PYTHON diff --cc tests/scripts/task_python_unittest.sh index 8472b142b8,511b3ec2a5..e5e5603429 --- a/tests/scripts/task_python_unittest.sh +++ b/tests/scripts/task_python_unittest.sh @@@ -18,12 -18,15 +18,7 @@@ set -euxo pipefail -export PYTHONPATH="$(pwd)/python" +source "$(dirname "$0")/task_python_install_wheels.sh" export PYTEST_ADDOPTS="${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" -# setup tvm-ffi into python folder -uv pip install -v --target=python ./3rdparty/tvm-ffi/ - --# LLVMModule JIT execution. Installed here as well as in the CI image because a docker/ --# change does not reach the test containers in the same run (they use the pinned tag). --# --no-deps keeps the submodule-built tvm-ffi above from being replaced by a PyPI build. --uv pip install --no-deps apache-tvm-ffi-orcjit==0.1.2 -- -python3 -m pytest -vvs -n auto -m "${TVM_TEST_MARKER:-not gpu}" tests/python +python3 -m pytest -vvs -n auto -m "${TVM_TEST_MARKER:-not gpu}" "${@:-tests/python}" diff --cc tests/scripts/task_python_unittest_gpuonly.sh index 9e4b95ec3d,ac9e112b84..9aa95cb201 --- a/tests/scripts/task_python_unittest_gpuonly.sh +++ b/tests/scripts/task_python_unittest_gpuonly.sh @@@ -31,4 -31,9 +31,6 @@@ export TVM_TEST_TARGETS='cuda;metal;roc # only minimal functionality of Vulkan runtime. export TVM_TEST_TARGETS='{"kind":"vulkan","from_device":0}' - ./tests/scripts/task_python_unittest.sh tests/python/codegen/test_target_codegen_vulkan.py -export PYTHONPATH="$(pwd)/python" -export PYTEST_ADDOPTS="${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" - -python3 -m pytest -vvs -n auto -m "${TVM_TEST_MARKER}" \ ++./tests/scripts/task_python_unittest.sh \ + tests/python/backend/vulkan/codegen/test_vulkan.py \ + tests/python/backend/vulkan/test_vulkan_module_export.py diff --cc tests/scripts/task_web_wasm.sh index 5f4d8d309e,5f4d8d309e..e2f6d6c4db --- a/tests/scripts/task_web_wasm.sh +++ b/tests/scripts/task_web_wasm.sh @@@ -18,10 -18,10 +18,8 @@@ set -euxo pipefail --export PYTHONPATH=`pwd`/python -- --# setup tvm-ffi into python folder --uv pip install -v --target=python ./3rdparty/tvm-ffi/ ++# Host Python uses the built wheel; Emscripten still builds its separate runtime. ++source "$(dirname "$0")/task_python_install_wheels.sh" rm -rf .emscripten_cache cd web
