This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch tvm-simplify-jenkins-pytest-sharding in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 0992b4955b2fd5d8e6447fd18095ef9b90c07fab Author: Tianqi Chen <[email protected]> AuthorDate: Sat Jul 4 23:45:42 2026 +0000 [CI] Simplify Jenkins pytest execution Run each filtered suite once and rely on pytest-xdist inside the direct launchers. Remove obsolete sharding selectors and pytest helpers while preserving GPU filtering and Jenkins JUnit reporting. --- ci/jenkins/generated/arm_jenkinsfile.groovy | 4 +- ci/jenkins/generated/cpu_jenkinsfile.groovy | 79 ++------------ ci/jenkins/generated/docker_jenkinsfile.groovy | 4 +- ci/jenkins/generated/gpu_jenkinsfile.groovy | 98 +++-------------- ci/jenkins/generated/wasm_jenkinsfile.groovy | 4 +- ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 | 3 +- ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 | 8 +- ci/jenkins/templates/utils/base.groovy.j2 | 2 +- ci/jenkins/templates/utils/macros.j2 | 15 +-- ci/scripts/jenkins/pytest_ids.py | 43 -------- ci/scripts/jenkins/pytest_wrapper.py | 137 ------------------------ conftest.py | 54 ---------- docker/install/ubuntu_install_python_package.sh | 2 - tests/scripts/task_python_unittest.sh | 9 +- tests/scripts/task_python_unittest_gpuonly.sh | 5 +- 15 files changed, 40 insertions(+), 427 deletions(-) diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy index d6826878bf..573885545a 100644 --- a/ci/jenkins/generated/arm_jenkinsfile.groovy +++ b/ci/jenkins/generated/arm_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.520193 +// Generated at 2026-07-04T23:53:56.833713 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy index 879cd8b79f..dc944096ab 100644 --- a/ci/jenkins/generated/cpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.505504 +// Generated at 2026-07-04T23:53:56.855205 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 @@ -510,52 +510,7 @@ build() - -def shard_run_unittest_CPU_1_of_2(node_type) { - echo 'Begin running on node_type ' + node_type - if (!skip_ci && is_docs_only_build != 1) { - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") { - // NOTE: if exception happens, it will be caught outside - init_git() - docker_init(ci_cpu) - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=cpu', - 'TEST_STEP_NAME=unittest: CPU', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=0', - "SKIP_SLOW_TESTS=${skip_slow_tests}"], { - sh( - script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu", - label: 'Download artifacts from S3', - ) - - ci_setup(ci_cpu) - cpp_unittest(ci_cpu) - python_unittest(ci_cpu) - }) - } - // only run upload if things are successful - try { - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU --items build/pytest-results", - label: 'Upload JUnits to S3', - ) - - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } - } - } - echo 'End running on node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('unittest: CPU 1 of 2') - } -} - -def shard_run_unittest_CPU_2_of_2(node_type) { +def run_unittest_CPU(node_type) { echo 'Begin running on node_type ' + node_type if (!skip_ci && is_docs_only_build != 1) { node(node_type) { @@ -567,8 +522,6 @@ def shard_run_unittest_CPU_2_of_2(node_type) { withEnv([ 'PLATFORM=cpu', 'TEST_STEP_NAME=unittest: CPU', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh( script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu", @@ -595,7 +548,7 @@ def shard_run_unittest_CPU_2_of_2(node_type) { } echo 'End running on node_type ' + node_type } else { - Utils.markStageSkippedForConditional('unittest: CPU 2 of 2') + Utils.markStageSkippedForConditional('unittest: CPU') } } @@ -606,27 +559,9 @@ def test() { SKIP_SLOW_TESTS = "${skip_slow_tests}" } parallel( - 'unittest: CPU 1 of 2': { - try { - shard_run_unittest_CPU_1_of_2('CPU-SMALL-SPOT') - } catch (Throwable ex) { - echo 'Exception during SPOT run ' + ex.toString() - if (is_last_build()) { - // retry if at last build - // mark the current stage as success - // and try again via on demand node - echo 'Retry on-demand given it is last build' - currentBuild.result = 'SUCCESS' - shard_run_unittest_CPU_1_of_2('CPU-SMALL') - } else { - echo 'Exit since it is not last build' - throw ex - } - } - }, - 'unittest: CPU 2 of 2': { + 'unittest: CPU': { try { - shard_run_unittest_CPU_2_of_2('CPU-SMALL-SPOT') + run_unittest_CPU('CPU-SMALL-SPOT') } catch (Throwable ex) { echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { @@ -635,7 +570,7 @@ def test() { // and try again via on demand node echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' - shard_run_unittest_CPU_2_of_2('CPU-SMALL') + run_unittest_CPU('CPU-SMALL') } else { echo 'Exit since it is not last build' throw ex diff --git a/ci/jenkins/generated/docker_jenkinsfile.groovy b/ci/jenkins/generated/docker_jenkinsfile.groovy index f5409e23aa..95f2fd8dd6 100644 --- a/ci/jenkins/generated/docker_jenkinsfile.groovy +++ b/ci/jenkins/generated/docker_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.487314 +// Generated at 2026-07-04T23:53:56.875051 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy index 28c9a6040e..d31160dd52 100644 --- a/ci/jenkins/generated/gpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.533751 +// Generated at 2026-07-04T23:53:56.896868 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 @@ -515,8 +515,7 @@ def build() { build() - -def shard_run_unittest_GPU_1_of_2(node_type) { +def run_unittest_GPU(node_type) { echo 'Begin running on node_type ' + node_type if (!skip_ci && is_docs_only_build != 1) { node(node_type) { @@ -528,58 +527,6 @@ def shard_run_unittest_GPU_1_of_2(node_type) { withEnv([ 'PLATFORM=gpu', 'TEST_STEP_NAME=unittest: GPU', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=0', - "SKIP_SLOW_TESTS=${skip_slow_tests}"], { - sh( - script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu", - label: 'Download artifacts from S3', - ) - - ci_setup(ci_gpu) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", - label: 'Run Python GPU unit tests', - ) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh", - label: 'Run Python GPU integration tests', - ) - }) - } - // only run upload if things are successful - try { - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items build/pytest-results", - label: 'Upload JUnits to S3', - ) - - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } - } - } - echo 'End running on node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('unittest: GPU 1 of 2') - } -} - -def shard_run_unittest_GPU_2_of_2(node_type) { - echo 'Begin running on node_type ' + node_type - if (!skip_ci && is_docs_only_build != 1) { - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { - // NOTE: if exception happens, it will be caught outside - init_git() - docker_init(ci_gpu) - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=gpu', - 'TEST_STEP_NAME=unittest: GPU', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh( script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu", @@ -616,14 +563,13 @@ def shard_run_unittest_GPU_2_of_2(node_type) { } echo 'End running on node_type ' + node_type } else { - Utils.markStageSkippedForConditional('unittest: GPU 2 of 2') + Utils.markStageSkippedForConditional('unittest: GPU') } } - -def shard_run_docs_GPU_1_of_1(node_type) { +def run_docs_GPU(node_type) { echo 'Begin running on node_type ' + node_type if (!skip_ci) { node(node_type) { @@ -635,8 +581,6 @@ def shard_run_docs_GPU_1_of_1(node_type) { withEnv([ 'PLATFORM=gpu', 'TEST_STEP_NAME=docs: GPU', - 'TVM_NUM_SHARDS=1', - 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh( script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu", @@ -674,7 +618,7 @@ def shard_run_docs_GPU_1_of_1(node_type) { } echo 'End running on node_type ' + node_type } else { - Utils.markStageSkippedForConditional('docs: GPU 1 of 1') + Utils.markStageSkippedForConditional('docs: GPU') } } @@ -686,27 +630,9 @@ def test() { SKIP_SLOW_TESTS = "${skip_slow_tests}" } parallel( - 'unittest: GPU 1 of 2': { - try { - shard_run_unittest_GPU_1_of_2('GPU-SPOT') - } catch (Throwable ex) { - echo 'Exception during SPOT run ' + ex.toString() - if (is_last_build()) { - // retry if at last build - // mark the current stage as success - // and try again via on demand node - echo 'Retry on-demand given it is last build' - currentBuild.result = 'SUCCESS' - shard_run_unittest_GPU_1_of_2('GPU') - } else { - echo 'Exit since it is not last build' - throw ex - } - } - }, - 'unittest: GPU 2 of 2': { + 'unittest: GPU': { try { - shard_run_unittest_GPU_2_of_2('GPU-SPOT') + run_unittest_GPU('GPU-SPOT') } catch (Throwable ex) { echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { @@ -715,16 +641,16 @@ def test() { // and try again via on demand node echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' - shard_run_unittest_GPU_2_of_2('GPU') + run_unittest_GPU('GPU') } else { echo 'Exit since it is not last build' throw ex } } }, - 'docs: GPU 1 of 1': { + 'docs: GPU': { try { - shard_run_docs_GPU_1_of_1('GPU-SPOT') + run_docs_GPU('GPU-SPOT') } catch (Throwable ex) { echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { @@ -733,7 +659,7 @@ def test() { // and try again via on demand node echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' - shard_run_docs_GPU_1_of_1('GPU') + run_docs_GPU('GPU') } else { echo 'Exit since it is not last build' throw ex diff --git a/ci/jenkins/generated/wasm_jenkinsfile.groovy b/ci/jenkins/generated/wasm_jenkinsfile.groovy index ab64444e4a..b6946a40fe 100644 --- a/ci/jenkins/generated/wasm_jenkinsfile.groovy +++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.550555 +// Generated at 2026-07-04T23:53:56.918767 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 diff --git a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 index d2e479d5e8..75b43c52a8 100644 --- a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 +++ b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 @@ -37,11 +37,10 @@ {% set test_method_names = [] %} -{% call(shard_index, num_shards) m.sharded_test_step( +{% call m.test_step( name="unittest: CPU", ws="tvm/ut-python-cpu", platform="cpu", - num_shards=2, docker_image="ci_cpu", test_method_names=test_method_names, ) %} diff --git a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 index 7ab5256419..33e5a8aed5 100644 --- a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 +++ b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 @@ -38,9 +38,8 @@ {% set test_method_names = [] %} -{% call(shard_index, num_shards) m.sharded_test_step( +{% call m.test_step( name="unittest: GPU", - num_shards=2, ws="tvm/ut-python-gpu", platform="gpu", docker_image="ci_gpu", @@ -48,12 +47,10 @@ ) %} {{ m.download_artifacts(tag='gpu') }} ci_setup(ci_gpu) - {% if shard_index == 2 or num_shards < 2 %} sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh", label: 'Run Java unit tests', ) - {% endif %} sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", label: 'Run Python GPU unit tests', @@ -65,9 +62,8 @@ {% endcall %} -{% call(shard_index, num_shards) m.sharded_test_step( +{% call m.test_step( name="docs: GPU", - num_shards=1, ws="tvm/docs-python-gpu", platform="gpu", docker_image="ci_gpu", diff --git a/ci/jenkins/templates/utils/base.groovy.j2 b/ci/jenkins/templates/utils/base.groovy.j2 index 813ceee886..a8fda460b7 100644 --- a/ci/jenkins/templates/utils/base.groovy.j2 +++ b/ci/jenkins/templates/utils/base.groovy.j2 @@ -82,7 +82,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 diff --git a/ci/jenkins/templates/utils/macros.j2 b/ci/jenkins/templates/utils/macros.j2 index b1bd3679ac..bd0e0d4ee4 100644 --- a/ci/jenkins/templates/utils/macros.j2 +++ b/ci/jenkins/templates/utils/macros.j2 @@ -26,10 +26,8 @@ sh( ) {% endmacro %} -{% macro sharded_test_step(name, num_shards, ws, docker_image, platform, test_method_names, condition="!skip_ci && is_docs_only_build != 1") %} - -{% for shard_index in range(1, num_shards + 1) %} -{% set method_name = "shard_run_" + name.replace(":", "").replace(" ", "-").replace("-", "_") + "_" + shard_index|string + "_of_" + num_shards|string %} +{% macro test_step(name, ws, docker_image, platform, test_method_names, condition="!skip_ci && is_docs_only_build != 1") %} +{% set method_name = "run_" + name.replace(":", "").replace(" ", "-").replace("-", "_") %} {% set test_dir_name = name.replace(":", "").replace(" ", "-").replace("-", "_")|string %} def {{ method_name }}(node_type) { echo 'Begin running on node_type ' + node_type @@ -43,10 +41,8 @@ def {{ method_name }}(node_type) { withEnv([ 'PLATFORM={{ platform }}', 'TEST_STEP_NAME={{ name }}', - 'TVM_NUM_SHARDS={{ num_shards }}', - 'TVM_SHARD_INDEX={{ shard_index - 1 }}', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { - {{ caller(shard_index, num_shards) | trim | indent(width=12) }} + {{ caller() | trim | indent(width=12) }} }) } // only run upload if things are successful @@ -60,12 +56,11 @@ def {{ method_name }}(node_type) { } echo 'End running on node_type ' + node_type } else { - Utils.markStageSkippedForConditional('{{ name }} {{ shard_index }} of {{ num_shards }}') + Utils.markStageSkippedForConditional('{{ name }}') } } -{% set _ = test_method_names.append((name + " " + shard_index|string + " of " + num_shards|string, method_name)) %} +{% set _ = test_method_names.append((name, method_name)) %} -{% endfor %} {% endmacro %} {% macro invoke_build(name, condition, node, docker_image, ws, platform) %} diff --git a/ci/scripts/jenkins/pytest_ids.py b/ci/scripts/jenkins/pytest_ids.py deleted file mode 100755 index 548069e855..0000000000 --- a/ci/scripts/jenkins/pytest_ids.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -import argparse -import io -from contextlib import redirect_stdout - -import pytest - - -class NodeidsCollector: - def pytest_collection_modifyitems(self, items): - self.nodeids = [item.nodeid for item in items] - - -def main(folder): - collector = NodeidsCollector() - f = io.StringIO() - with redirect_stdout(f): - pytest.main(["-qq", "--collect-only", folder], plugins=[collector]) - for nodeid in collector.nodeids: - print(nodeid) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="List pytest nodeids for a folder") - parser.add_argument("--folder", required=True, help="test folder to inspect") - args = parser.parse_args() - main(args.folder) diff --git a/ci/scripts/jenkins/pytest_wrapper.py b/ci/scripts/jenkins/pytest_wrapper.py deleted file mode 100755 index a667810293..0000000000 --- a/ci/scripts/jenkins/pytest_wrapper.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ruff: noqa: E501 -import argparse -import logging -import os -import textwrap -import urllib.parse -from pathlib import Path - -import junitparser -from cmd_utils import init_log - -REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent - - -def lstrip(s: str, prefix: str) -> str: - if s.startswith(prefix): - s = s[len(prefix) :] - return s - - -def classname_to_file(classname: str) -> str: - classname = lstrip(classname, "cython.") - classname = lstrip(classname, "ctypes.") - return classname.replace(".", "/") + ".py" - - -def failed_test_ids() -> list[str]: - FAILURE_TYPES = (junitparser.Failure, junitparser.Error) - junit_dir = REPO_ROOT / "build" / "pytest-results" - failed_node_ids = [] - for junit in junit_dir.glob("*.xml"): - xml = junitparser.JUnitXml.fromfile(str(junit)) - for suite in xml: - # handle suites - for case in suite: - if case.result is None: - logging.warn(f"Incorrectly formatted JUnit found, result was None on {case}") - continue - - if len(case.result) > 0 and isinstance(case.result[0], FAILURE_TYPES): - node_id = classname_to_file(case.classname) + "::" + case.name - failed_node_ids.append(node_id) - - return list(set(failed_node_ids)) - - -def repro_command(build_type: str, failed_node_ids: list[str]) -> str | None: - """ - Parse available JUnit XML files and output a command that users can run to - reproduce CI failures locally - """ - test_args = [f"--tests {node_id}" for node_id in failed_node_ids] - test_args_str = " ".join(test_args) - return f"python3 tests/scripts/ci.py {build_type} {test_args_str}" - - -def make_issue_url(failed_node_ids: list[str]) -> str: - names = [f"`{node_id}`" for node_id in failed_node_ids] - run_url = os.getenv("RUN_DISPLAY_URL", "<insert run URL>") - test_bullets = [f" - `{node_id}`" for node_id in failed_node_ids] - params = { - "labels": "test: flaky", - "title": "[Flaky Test] " + ", ".join(names), - "body": textwrap.dedent( - """ - These tests were found to be flaky (intermittently failing on `main` or failed in a PR with unrelated changes). See [the docs](https://github.com/apache/tvm/blob/main/docs/contribute/ci.rst#handling-flaky-failures) for details. - - ### Tests(s)\n - """ - ) - + "\n".join(test_bullets) - + f"\n\n### Jenkins Links\n\n - {run_url}", - } - return "https://github.com/apache/tvm/issues/new?"; + urllib.parse.urlencode(params) - - -def show_failure_help(failed_suites: list[str]) -> None: - failed_node_ids = failed_test_ids() - - if len(failed_node_ids) == 0: - return - - build_type = os.getenv("PLATFORM") - - if build_type is None: - raise RuntimeError("build type was None, cannot show command") - - repro = repro_command(build_type=build_type, failed_node_ids=failed_node_ids) - if repro is None: - print("No test failures detected") - return - - print(f"Report flaky test shortcut: {make_issue_url(failed_node_ids)}") - print("=============================== PYTEST FAILURES ================================") - print( - "These pytest suites failed to execute. The results can be found in the " - "Jenkins 'Tests' tab or by scrolling up through the raw logs here. " - "If there is no test listed below, the failure likely came from a segmentation " - "fault which you can find in the logs above.\n" - ) - if failed_suites is not None and len(failed_suites) > 0: - print("\n".join([f" - {suite}" for suite in failed_suites])) - print("") - - print("You can reproduce these specific failures locally with this command:\n") - print(textwrap.indent(repro, prefix=" ")) - print("") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Print information about a failed pytest run") - args, other = parser.parse_known_args() - init_log() - - try: - show_failure_help(failed_suites=other) - except Exception as e: - # This script shouldn't ever introduce failures since it's just there to - # add extra information, so ignore any errors - logging.exception(e) diff --git a/conftest.py b/conftest.py index dd32937620..dd294ea548 100644 --- a/conftest.py +++ b/conftest.py @@ -14,8 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# ruff: noqa: E501 -import hashlib import os import sys from pathlib import Path @@ -25,58 +23,6 @@ IS_IN_CI = os.getenv("CI", "") == "true" REPO_ROOT = Path(__file__).resolve().parent -# These are long running tests (manually curated and extracted from CI logs) -# that should be allocated to test shards in a round-robin fashion. These are -# taken from the 20 (arbitrary number) of tests as from -# https://ci.tlcpack.ai/job/tvm/job/main/2907/testReport -_slowest_tests = [] -HARDCODED_ALLOCATIONS = {} -for idx, test in enumerate(_slowest_tests): - HARDCODED_ALLOCATIONS[test] = idx - -# These rely on running on the same node to pass successfully -FIXED_ALLOCATION_PREFIXES = { - "tests/python/testing/test_tvm_testing_features.py": 0, -} - - -def find_shard_index(nodeid: str, num_shards: int) -> int: - """ - Return the index of the shard that should run this test - """ - for prefix, target_shard_idx in FIXED_ALLOCATION_PREFIXES.items(): - if nodeid.startswith(prefix): - if target_shard_idx >= num_shards: - raise RuntimeError( - f"Cannot collect sharded tests, {nodeid} has hardcoded shard index {target_shard_idx} among only {num_shards} shards" - ) - return target_shard_idx - - if nodeid in HARDCODED_ALLOCATIONS: - hash = HARDCODED_ALLOCATIONS[nodeid] - else: - hash = hashlib.md5(nodeid.encode()) - hash = int(hash.hexdigest(), 16) - - return hash % num_shards - - -def pytest_collection_modifyitems(config, items): - if not all(k in os.environ for k in ["CI", "TVM_NUM_SHARDS", "TVM_SHARD_INDEX"]): - # Only apportion tests if in CI and in a job that is set up for it - return - - num_shards = int(os.environ["TVM_NUM_SHARDS"]) - shard_index = int(os.environ["TVM_SHARD_INDEX"]) - - print(f"Marking tests for shard {shard_index} of {num_shards}") - items_copy = list(items) - for item in items_copy: - item_shard_index = find_shard_index(item.nodeid, num_shards=num_shards) - if item_shard_index != shard_index: - items.remove(item) - - def pytest_sessionstart(): if IS_IN_CI: hook_script_dir = REPO_ROOT / "tests" / "scripts" / "request_hook" diff --git a/docker/install/ubuntu_install_python_package.sh b/docker/install/ubuntu_install_python_package.sh index d07f19ed3c..4445dfd83e 100755 --- a/docker/install/ubuntu_install_python_package.sh +++ b/docker/install/ubuntu_install_python_package.sh @@ -32,13 +32,11 @@ uv pip install --upgrade \ Pillow==12.1.1 \ "psutil~=7.0" \ "pytest~=8.3" \ - "pytest-profiling~=1.8" \ "pytest-xdist~=3.6" \ pytest-rerunfailures==16.1 \ "requests~=2.32" \ "scipy~=1.13" \ "Jinja2~=3.1" \ - junitparser==4.0.2 \ "six~=1.17" \ "tornado~=6.4" \ "ml_dtypes~=0.5" \ diff --git a/tests/scripts/task_python_unittest.sh b/tests/scripts/task_python_unittest.sh index 14675bf8a0..30e5bb2bf3 100755 --- a/tests/scripts/task_python_unittest.sh +++ b/tests/scripts/task_python_unittest.sh @@ -32,12 +32,11 @@ uv pip install -v --target=python ./3rdparty/tvm-ffi/ if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then TVM_UNITTEST_TESTSUITE_NAME=python-unittest fi -PYTEST_SHARD_SUFFIX="${TVM_SHARD_INDEX:+-shard-${TVM_SHARD_INDEX}}" # First run the minimal platform test. A GPU-only run can select no tests here. python3 -m pytest -n auto \ - -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test${PYTEST_SHARD_SUFFIX}" \ - "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test${PYTEST_SHARD_SUFFIX}.xml" \ + -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test" \ + "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test.xml" \ --junit-prefix=cython \ tests/python/all-platform-minimal-test || [ "$?" -eq 5 ] @@ -73,7 +72,7 @@ for TEST_FILE in "${TEST_FILES[@]}"; do done python3 -m pytest -n auto --dist=loadgroup \ - -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}${PYTEST_SHARD_SUFFIX}" \ - "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}${PYTEST_SHARD_SUFFIX}.xml" \ + -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}" \ + "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}.xml" \ --junit-prefix=cython \ "${PYTEST_TARGETS[@]}" diff --git a/tests/scripts/task_python_unittest_gpuonly.sh b/tests/scripts/task_python_unittest_gpuonly.sh index 9753a1a8a4..6bbd85dc10 100755 --- a/tests/scripts/task_python_unittest_gpuonly.sh +++ b/tests/scripts/task_python_unittest_gpuonly.sh @@ -36,10 +36,9 @@ export TVM_UNITTEST_TESTSUITE_NAME=python-codegen-vulkan export PYTHONPATH="$(pwd)/python" export PYTEST_ADDOPTS="-s -vv ${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" mkdir -p build/pytest-results -PYTEST_SHARD_SUFFIX="${TVM_SHARD_INDEX:+-shard-${TVM_SHARD_INDEX}}" python3 -m pytest -n auto \ - -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}${PYTEST_SHARD_SUFFIX}" \ - "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}${PYTEST_SHARD_SUFFIX}.xml" \ + -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}" \ + "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}.xml" \ --junit-prefix=cython \ tests/python/codegen/test_target_codegen_vulkan.py
