This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch tvm-simplify-jenkins-pytest-sharding in repository https://gitbox.apache.org/repos/asf/tvm.git
commit a3048810fb71c0907c18642eb47ee8d6b09fdc77 Author: Tianqi Chen <[email protected]> AuthorDate: Sun Jul 5 02:14:27 2026 +0000 [CI] Simplify Jenkins pytest execution Enable standard pytest-xdist automatic parallelism in the canonical launchers while removing external sharding, obsolete wrapper machinery, JUnit reporting, and its XML consumer. Repair broad-suite targets and failure propagation, and bound active nested MetaSchedule test parallelism without blanket serialization. --- .github/workflows/main.yml | 2 +- ci/jenkins/generated/arm_jenkinsfile.groovy | 11 +- ci/jenkins/generated/cpu_jenkinsfile.groovy | 98 ++------------- ci/jenkins/generated/docker_jenkinsfile.groovy | 11 +- ci/jenkins/generated/gpu_jenkinsfile.groovy | 129 ++----------------- ci/jenkins/generated/wasm_jenkinsfile.groovy | 12 +- ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 | 4 +- ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 | 10 +- ci/jenkins/templates/utils/Build.groovy.j2 | 7 -- ci/jenkins/templates/utils/base.groovy.j2 | 2 +- ci/jenkins/templates/utils/macros.j2 | 30 +---- ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 | 1 - ci/scripts/jenkins/pytest_ids.py | 43 ------- ci/scripts/jenkins/pytest_wrapper.py | 137 --------------------- conftest.py | 54 -------- docker/install/ubuntu_install_python_package.sh | 2 - docs/contribute/pull_request.rst | 2 +- python/tvm/testing/plugin.py | 9 -- .../meta_schedule/test_meta_schedule_cost_model.py | 11 +- .../test_meta_schedule_search_strategy.py | 1 + .../test_meta_schedule_task_scheduler.py | 13 ++ .../test_s_tir_transform_lower_match_buffer.py | 2 +- tests/python/testing/test_tvm_testing_features.py | 3 +- tests/scripts/setup-pytest-env.sh | 98 --------------- tests/scripts/task_clear_pytest.sh | 8 +- tests/scripts/task_python_docs.sh | 2 +- tests/scripts/task_python_integration.sh | 2 +- tests/scripts/task_python_nightly.sh | 33 ----- tests/scripts/task_python_unittest.sh | 29 +++-- tests/scripts/task_python_unittest_gpuonly.sh | 11 +- 30 files changed, 87 insertions(+), 690 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a0e4cb0490..54dc54316d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,7 +69,7 @@ jobs: python -m pytest -v -s 'tests/python/codegen/test_target_codegen_metal.py::test_func_with_trailing_pod_params' - name: Minimal Metal Compile-and-Run shell: bash -l {0} - run: >- + run: | python -m pytest -v -s 'tests/python/codegen/test_target_codegen_metal.py' python -m pytest -v -s 'tests/python/codegen/test_target_codegen_gpu_common.py' python -m pytest -v -s 'tests/python/codegen/test_gpu_codegen_allreduce.py::test_allreduce_sum[dims0-metal]' diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy index d6826878bf..3442caded7 100644 --- a/ci/jenkins/generated/arm_jenkinsfile.groovy +++ b/ci/jenkins/generated/arm_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.520193 +// Generated at 2026-07-05T01:05:57.719405 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 @@ -404,13 +404,6 @@ def prepare(node_type) { } } } -def ci_setup(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh", - label: 'Clean up old workspace', - ) -} - def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy index 879cd8b79f..0d3e408235 100644 --- a/ci/jenkins/generated/cpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.505504 +// Generated at 2026-07-05T01:05:57.741181 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 @@ -404,13 +404,6 @@ def prepare(node_type) { } } } -def ci_setup(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh", - label: 'Clean up old workspace', - ) -} - def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", @@ -510,52 +503,7 @@ build() - -def shard_run_unittest_CPU_1_of_2(node_type) { - echo 'Begin running on node_type ' + node_type - if (!skip_ci && is_docs_only_build != 1) { - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") { - // NOTE: if exception happens, it will be caught outside - init_git() - docker_init(ci_cpu) - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=cpu', - 'TEST_STEP_NAME=unittest: CPU', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=0', - "SKIP_SLOW_TESTS=${skip_slow_tests}"], { - sh( - script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu", - label: 'Download artifacts from S3', - ) - - ci_setup(ci_cpu) - cpp_unittest(ci_cpu) - python_unittest(ci_cpu) - }) - } - // only run upload if things are successful - try { - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU --items build/pytest-results", - label: 'Upload JUnits to S3', - ) - - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } - } - } - echo 'End running on node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('unittest: CPU 1 of 2') - } -} - -def shard_run_unittest_CPU_2_of_2(node_type) { +def run_unittest_CPU(node_type) { echo 'Begin running on node_type ' + node_type if (!skip_ci && is_docs_only_build != 1) { node(node_type) { @@ -567,35 +515,21 @@ def shard_run_unittest_CPU_2_of_2(node_type) { withEnv([ 'PLATFORM=cpu', 'TEST_STEP_NAME=unittest: CPU', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh( script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu", label: 'Download artifacts from S3', ) - ci_setup(ci_cpu) cpp_unittest(ci_cpu) python_unittest(ci_cpu) }) } - // only run upload if things are successful - try { - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU --items build/pytest-results", - label: 'Upload JUnits to S3', - ) - - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } } } echo 'End running on node_type ' + node_type } else { - Utils.markStageSkippedForConditional('unittest: CPU 2 of 2') + Utils.markStageSkippedForConditional('unittest: CPU') } } @@ -606,27 +540,9 @@ def test() { SKIP_SLOW_TESTS = "${skip_slow_tests}" } parallel( - 'unittest: CPU 1 of 2': { - try { - shard_run_unittest_CPU_1_of_2('CPU-SMALL-SPOT') - } catch (Throwable ex) { - echo 'Exception during SPOT run ' + ex.toString() - if (is_last_build()) { - // retry if at last build - // mark the current stage as success - // and try again via on demand node - echo 'Retry on-demand given it is last build' - currentBuild.result = 'SUCCESS' - shard_run_unittest_CPU_1_of_2('CPU-SMALL') - } else { - echo 'Exit since it is not last build' - throw ex - } - } - }, - 'unittest: CPU 2 of 2': { + 'unittest: CPU': { try { - shard_run_unittest_CPU_2_of_2('CPU-SMALL-SPOT') + run_unittest_CPU('CPU-SMALL-SPOT') } catch (Throwable ex) { echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { @@ -635,7 +551,7 @@ def test() { // and try again via on demand node echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' - shard_run_unittest_CPU_2_of_2('CPU-SMALL') + run_unittest_CPU('CPU-SMALL') } else { echo 'Exit since it is not last build' throw ex diff --git a/ci/jenkins/generated/docker_jenkinsfile.groovy b/ci/jenkins/generated/docker_jenkinsfile.groovy index f5409e23aa..8ffe78942a 100644 --- a/ci/jenkins/generated/docker_jenkinsfile.groovy +++ b/ci/jenkins/generated/docker_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.487314 +// Generated at 2026-07-05T01:05:57.759905 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 @@ -404,13 +404,6 @@ def prepare(node_type) { } } } -def ci_setup(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh", - label: 'Clean up old workspace', - ) -} - def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy index 28c9a6040e..792b7eadc4 100644 --- a/ci/jenkins/generated/gpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.533751 +// Generated at 2026-07-05T01:05:57.780838 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 @@ -404,13 +404,6 @@ def prepare(node_type) { } } } -def ci_setup(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh", - label: 'Clean up old workspace', - ) -} - def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", @@ -515,8 +508,7 @@ def build() { build() - -def shard_run_unittest_GPU_1_of_2(node_type) { +def run_unittest_GPU(node_type) { echo 'Begin running on node_type ' + node_type if (!skip_ci && is_docs_only_build != 1) { node(node_type) { @@ -528,65 +520,12 @@ def shard_run_unittest_GPU_1_of_2(node_type) { withEnv([ 'PLATFORM=gpu', 'TEST_STEP_NAME=unittest: GPU', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh( script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu", label: 'Download artifacts from S3', ) - ci_setup(ci_gpu) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", - label: 'Run Python GPU unit tests', - ) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh", - label: 'Run Python GPU integration tests', - ) - }) - } - // only run upload if things are successful - try { - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items build/pytest-results", - label: 'Upload JUnits to S3', - ) - - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } - } - } - echo 'End running on node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('unittest: GPU 1 of 2') - } -} - -def shard_run_unittest_GPU_2_of_2(node_type) { - echo 'Begin running on node_type ' + node_type - if (!skip_ci && is_docs_only_build != 1) { - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { - // NOTE: if exception happens, it will be caught outside - init_git() - docker_init(ci_gpu) - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=gpu', - 'TEST_STEP_NAME=unittest: GPU', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=1', - "SKIP_SLOW_TESTS=${skip_slow_tests}"], { - sh( - script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu", - label: 'Download artifacts from S3', - ) - - ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh", label: 'Run Java unit tests', @@ -601,29 +540,17 @@ def shard_run_unittest_GPU_2_of_2(node_type) { ) }) } - // only run upload if things are successful - try { - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items build/pytest-results", - label: 'Upload JUnits to S3', - ) - - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } } } echo 'End running on node_type ' + node_type } else { - Utils.markStageSkippedForConditional('unittest: GPU 2 of 2') + Utils.markStageSkippedForConditional('unittest: GPU') } } - -def shard_run_docs_GPU_1_of_1(node_type) { +def run_docs_GPU(node_type) { echo 'Begin running on node_type ' + node_type if (!skip_ci) { node(node_type) { @@ -635,15 +562,12 @@ def shard_run_docs_GPU_1_of_1(node_type) { withEnv([ 'PLATFORM=gpu', 'TEST_STEP_NAME=docs: GPU', - 'TVM_NUM_SHARDS=1', - 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh( script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu", label: 'Download artifacts from S3', ) - ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh", label: 'Build docs', @@ -659,22 +583,11 @@ def shard_run_docs_GPU_1_of_1(node_type) { ) }) } - // only run upload if things are successful - try { - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/docs_GPU --items build/pytest-results", - label: 'Upload JUnits to S3', - ) - - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } } } echo 'End running on node_type ' + node_type } else { - Utils.markStageSkippedForConditional('docs: GPU 1 of 1') + Utils.markStageSkippedForConditional('docs: GPU') } } @@ -686,27 +599,9 @@ def test() { SKIP_SLOW_TESTS = "${skip_slow_tests}" } parallel( - 'unittest: GPU 1 of 2': { - try { - shard_run_unittest_GPU_1_of_2('GPU-SPOT') - } catch (Throwable ex) { - echo 'Exception during SPOT run ' + ex.toString() - if (is_last_build()) { - // retry if at last build - // mark the current stage as success - // and try again via on demand node - echo 'Retry on-demand given it is last build' - currentBuild.result = 'SUCCESS' - shard_run_unittest_GPU_1_of_2('GPU') - } else { - echo 'Exit since it is not last build' - throw ex - } - } - }, - 'unittest: GPU 2 of 2': { + 'unittest: GPU': { try { - shard_run_unittest_GPU_2_of_2('GPU-SPOT') + run_unittest_GPU('GPU-SPOT') } catch (Throwable ex) { echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { @@ -715,16 +610,16 @@ def test() { // and try again via on demand node echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' - shard_run_unittest_GPU_2_of_2('GPU') + run_unittest_GPU('GPU') } else { echo 'Exit since it is not last build' throw ex } } }, - 'docs: GPU 1 of 1': { + 'docs: GPU': { try { - shard_run_docs_GPU_1_of_1('GPU-SPOT') + run_docs_GPU('GPU-SPOT') } catch (Throwable ex) { echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { @@ -733,7 +628,7 @@ def test() { // and try again via on demand node echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' - shard_run_docs_GPU_1_of_1('GPU') + run_docs_GPU('GPU') } else { echo 'Exit since it is not last build' throw ex diff --git a/ci/jenkins/generated/wasm_jenkinsfile.groovy b/ci/jenkins/generated/wasm_jenkinsfile.groovy index ab64444e4a..7421fa86ca 100644 --- a/ci/jenkins/generated/wasm_jenkinsfile.groovy +++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.550555 +// Generated at 2026-07-05T01:05:57.802104 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 @@ -404,13 +404,6 @@ def prepare(node_type) { } } } -def ci_setup(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh", - label: 'Clean up old workspace', - ) -} - def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", @@ -471,7 +464,6 @@ def run_build(node_type) { cmake_build(ci_wasm, 'build') make_cpp_tests(ci_wasm, 'build') cpp_unittest(ci_wasm) - ci_setup(ci_wasm) sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh", label: 'Run WASM lint and tests', diff --git a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 index d2e479d5e8..62b7c3dfea 100644 --- a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 +++ b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 @@ -37,16 +37,14 @@ {% set test_method_names = [] %} -{% call(shard_index, num_shards) m.sharded_test_step( +{% call m.test_step( name="unittest: CPU", ws="tvm/ut-python-cpu", platform="cpu", - num_shards=2, docker_image="ci_cpu", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='cpu') }} - ci_setup(ci_cpu) cpp_unittest(ci_cpu) python_unittest(ci_cpu) {% endcall %} diff --git a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 index 7ab5256419..3a13dfe7f3 100644 --- a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 +++ b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 @@ -38,22 +38,18 @@ {% set test_method_names = [] %} -{% call(shard_index, num_shards) m.sharded_test_step( +{% call m.test_step( name="unittest: GPU", - num_shards=2, ws="tvm/ut-python-gpu", platform="gpu", docker_image="ci_gpu", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='gpu') }} - ci_setup(ci_gpu) - {% if shard_index == 2 or num_shards < 2 %} sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh", label: 'Run Java unit tests', ) - {% endif %} sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", label: 'Run Python GPU unit tests', @@ -65,9 +61,8 @@ {% endcall %} -{% call(shard_index, num_shards) m.sharded_test_step( +{% call m.test_step( name="docs: GPU", - num_shards=1, ws="tvm/docs-python-gpu", platform="gpu", docker_image="ci_gpu", @@ -75,7 +70,6 @@ test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='gpu') }} - ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh", label: 'Build docs', diff --git a/ci/jenkins/templates/utils/Build.groovy.j2 b/ci/jenkins/templates/utils/Build.groovy.j2 index c20a1c5c43..42bd4a991f 100644 --- a/ci/jenkins/templates/utils/Build.groovy.j2 +++ b/ci/jenkins/templates/utils/Build.groovy.j2 @@ -1,10 +1,3 @@ -def ci_setup(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh", - label: 'Clean up old workspace', - ) -} - def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", diff --git a/ci/jenkins/templates/utils/base.groovy.j2 b/ci/jenkins/templates/utils/base.groovy.j2 index 813ceee886..a8fda460b7 100644 --- a/ci/jenkins/templates/utils/base.groovy.j2 +++ b/ci/jenkins/templates/utils/base.groovy.j2 @@ -82,7 +82,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 diff --git a/ci/jenkins/templates/utils/macros.j2 b/ci/jenkins/templates/utils/macros.j2 index b1bd3679ac..959d91725f 100644 --- a/ci/jenkins/templates/utils/macros.j2 +++ b/ci/jenkins/templates/utils/macros.j2 @@ -19,18 +19,8 @@ "workspace/exec_${env.EXECUTOR_NUMBER}/{{ folder }}" {%- endmacro -%} -{% macro junit_to_s3(test_dir_name) %} -sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/{{ test_dir_name }} --items build/pytest-results", - label: 'Upload JUnits to S3', - ) -{% endmacro %} - -{% macro sharded_test_step(name, num_shards, ws, docker_image, platform, test_method_names, condition="!skip_ci && is_docs_only_build != 1") %} - -{% for shard_index in range(1, num_shards + 1) %} -{% set method_name = "shard_run_" + name.replace(":", "").replace(" ", "-").replace("-", "_") + "_" + shard_index|string + "_of_" + num_shards|string %} -{% set test_dir_name = name.replace(":", "").replace(" ", "-").replace("-", "_")|string %} +{% macro test_step(name, ws, docker_image, platform, test_method_names, condition="!skip_ci && is_docs_only_build != 1") %} +{% set method_name = "run_" + name.replace(":", "").replace(" ", "-").replace("-", "_") %} def {{ method_name }}(node_type) { echo 'Begin running on node_type ' + node_type if ({{ condition }}) { @@ -43,29 +33,19 @@ def {{ method_name }}(node_type) { withEnv([ 'PLATFORM={{ platform }}', 'TEST_STEP_NAME={{ name }}', - 'TVM_NUM_SHARDS={{ num_shards }}', - 'TVM_SHARD_INDEX={{ shard_index - 1 }}', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { - {{ caller(shard_index, num_shards) | trim | indent(width=12) }} + {{ caller() | trim | indent(width=12) }} }) } - // only run upload if things are successful - try { - {{ junit_to_s3(test_dir_name) }} - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } } } echo 'End running on node_type ' + node_type } else { - Utils.markStageSkippedForConditional('{{ name }} {{ shard_index }} of {{ num_shards }}') + Utils.markStageSkippedForConditional('{{ name }}') } } -{% set _ = test_method_names.append((name + " " + shard_index|string + " of " + num_shards|string, method_name)) %} +{% set _ = test_method_names.append((name, method_name)) %} -{% endfor %} {% endmacro %} {% macro invoke_build(name, condition, node, docker_image, ws, platform) %} diff --git a/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 b/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 index 0d119f043e..8e7d909bd1 100644 --- a/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 +++ b/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 @@ -32,7 +32,6 @@ cmake_build(ci_wasm, 'build') make_cpp_tests(ci_wasm, 'build') cpp_unittest(ci_wasm) - ci_setup(ci_wasm) sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh", label: 'Run WASM lint and tests', diff --git a/ci/scripts/jenkins/pytest_ids.py b/ci/scripts/jenkins/pytest_ids.py deleted file mode 100755 index 548069e855..0000000000 --- a/ci/scripts/jenkins/pytest_ids.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -import argparse -import io -from contextlib import redirect_stdout - -import pytest - - -class NodeidsCollector: - def pytest_collection_modifyitems(self, items): - self.nodeids = [item.nodeid for item in items] - - -def main(folder): - collector = NodeidsCollector() - f = io.StringIO() - with redirect_stdout(f): - pytest.main(["-qq", "--collect-only", folder], plugins=[collector]) - for nodeid in collector.nodeids: - print(nodeid) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="List pytest nodeids for a folder") - parser.add_argument("--folder", required=True, help="test folder to inspect") - args = parser.parse_args() - main(args.folder) diff --git a/ci/scripts/jenkins/pytest_wrapper.py b/ci/scripts/jenkins/pytest_wrapper.py deleted file mode 100755 index a667810293..0000000000 --- a/ci/scripts/jenkins/pytest_wrapper.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ruff: noqa: E501 -import argparse -import logging -import os -import textwrap -import urllib.parse -from pathlib import Path - -import junitparser -from cmd_utils import init_log - -REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent - - -def lstrip(s: str, prefix: str) -> str: - if s.startswith(prefix): - s = s[len(prefix) :] - return s - - -def classname_to_file(classname: str) -> str: - classname = lstrip(classname, "cython.") - classname = lstrip(classname, "ctypes.") - return classname.replace(".", "/") + ".py" - - -def failed_test_ids() -> list[str]: - FAILURE_TYPES = (junitparser.Failure, junitparser.Error) - junit_dir = REPO_ROOT / "build" / "pytest-results" - failed_node_ids = [] - for junit in junit_dir.glob("*.xml"): - xml = junitparser.JUnitXml.fromfile(str(junit)) - for suite in xml: - # handle suites - for case in suite: - if case.result is None: - logging.warn(f"Incorrectly formatted JUnit found, result was None on {case}") - continue - - if len(case.result) > 0 and isinstance(case.result[0], FAILURE_TYPES): - node_id = classname_to_file(case.classname) + "::" + case.name - failed_node_ids.append(node_id) - - return list(set(failed_node_ids)) - - -def repro_command(build_type: str, failed_node_ids: list[str]) -> str | None: - """ - Parse available JUnit XML files and output a command that users can run to - reproduce CI failures locally - """ - test_args = [f"--tests {node_id}" for node_id in failed_node_ids] - test_args_str = " ".join(test_args) - return f"python3 tests/scripts/ci.py {build_type} {test_args_str}" - - -def make_issue_url(failed_node_ids: list[str]) -> str: - names = [f"`{node_id}`" for node_id in failed_node_ids] - run_url = os.getenv("RUN_DISPLAY_URL", "<insert run URL>") - test_bullets = [f" - `{node_id}`" for node_id in failed_node_ids] - params = { - "labels": "test: flaky", - "title": "[Flaky Test] " + ", ".join(names), - "body": textwrap.dedent( - """ - These tests were found to be flaky (intermittently failing on `main` or failed in a PR with unrelated changes). See [the docs](https://github.com/apache/tvm/blob/main/docs/contribute/ci.rst#handling-flaky-failures) for details. - - ### Tests(s)\n - """ - ) - + "\n".join(test_bullets) - + f"\n\n### Jenkins Links\n\n - {run_url}", - } - return "https://github.com/apache/tvm/issues/new?"; + urllib.parse.urlencode(params) - - -def show_failure_help(failed_suites: list[str]) -> None: - failed_node_ids = failed_test_ids() - - if len(failed_node_ids) == 0: - return - - build_type = os.getenv("PLATFORM") - - if build_type is None: - raise RuntimeError("build type was None, cannot show command") - - repro = repro_command(build_type=build_type, failed_node_ids=failed_node_ids) - if repro is None: - print("No test failures detected") - return - - print(f"Report flaky test shortcut: {make_issue_url(failed_node_ids)}") - print("=============================== PYTEST FAILURES ================================") - print( - "These pytest suites failed to execute. The results can be found in the " - "Jenkins 'Tests' tab or by scrolling up through the raw logs here. " - "If there is no test listed below, the failure likely came from a segmentation " - "fault which you can find in the logs above.\n" - ) - if failed_suites is not None and len(failed_suites) > 0: - print("\n".join([f" - {suite}" for suite in failed_suites])) - print("") - - print("You can reproduce these specific failures locally with this command:\n") - print(textwrap.indent(repro, prefix=" ")) - print("") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Print information about a failed pytest run") - args, other = parser.parse_known_args() - init_log() - - try: - show_failure_help(failed_suites=other) - except Exception as e: - # This script shouldn't ever introduce failures since it's just there to - # add extra information, so ignore any errors - logging.exception(e) diff --git a/conftest.py b/conftest.py index dd32937620..dd294ea548 100644 --- a/conftest.py +++ b/conftest.py @@ -14,8 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# ruff: noqa: E501 -import hashlib import os import sys from pathlib import Path @@ -25,58 +23,6 @@ IS_IN_CI = os.getenv("CI", "") == "true" REPO_ROOT = Path(__file__).resolve().parent -# These are long running tests (manually curated and extracted from CI logs) -# that should be allocated to test shards in a round-robin fashion. These are -# taken from the 20 (arbitrary number) of tests as from -# https://ci.tlcpack.ai/job/tvm/job/main/2907/testReport -_slowest_tests = [] -HARDCODED_ALLOCATIONS = {} -for idx, test in enumerate(_slowest_tests): - HARDCODED_ALLOCATIONS[test] = idx - -# These rely on running on the same node to pass successfully -FIXED_ALLOCATION_PREFIXES = { - "tests/python/testing/test_tvm_testing_features.py": 0, -} - - -def find_shard_index(nodeid: str, num_shards: int) -> int: - """ - Return the index of the shard that should run this test - """ - for prefix, target_shard_idx in FIXED_ALLOCATION_PREFIXES.items(): - if nodeid.startswith(prefix): - if target_shard_idx >= num_shards: - raise RuntimeError( - f"Cannot collect sharded tests, {nodeid} has hardcoded shard index {target_shard_idx} among only {num_shards} shards" - ) - return target_shard_idx - - if nodeid in HARDCODED_ALLOCATIONS: - hash = HARDCODED_ALLOCATIONS[nodeid] - else: - hash = hashlib.md5(nodeid.encode()) - hash = int(hash.hexdigest(), 16) - - return hash % num_shards - - -def pytest_collection_modifyitems(config, items): - if not all(k in os.environ for k in ["CI", "TVM_NUM_SHARDS", "TVM_SHARD_INDEX"]): - # Only apportion tests if in CI and in a job that is set up for it - return - - num_shards = int(os.environ["TVM_NUM_SHARDS"]) - shard_index = int(os.environ["TVM_SHARD_INDEX"]) - - print(f"Marking tests for shard {shard_index} of {num_shards}") - items_copy = list(items) - for item in items_copy: - item_shard_index = find_shard_index(item.nodeid, num_shards=num_shards) - if item_shard_index != shard_index: - items.remove(item) - - def pytest_sessionstart(): if IS_IN_CI: hook_script_dir = REPO_ROOT / "tests" / "scripts" / "request_hook" diff --git a/docker/install/ubuntu_install_python_package.sh b/docker/install/ubuntu_install_python_package.sh index d07f19ed3c..4445dfd83e 100755 --- a/docker/install/ubuntu_install_python_package.sh +++ b/docker/install/ubuntu_install_python_package.sh @@ -32,13 +32,11 @@ uv pip install --upgrade \ Pillow==12.1.1 \ "psutil~=7.0" \ "pytest~=8.3" \ - "pytest-profiling~=1.8" \ "pytest-xdist~=3.6" \ pytest-rerunfailures==16.1 \ "requests~=2.32" \ "scipy~=1.13" \ "Jinja2~=3.1" \ - junitparser==4.0.2 \ "six~=1.17" \ "tornado~=6.4" \ "ml_dtypes~=0.5" \ diff --git a/docs/contribute/pull_request.rst b/docs/contribute/pull_request.rst index fe960c709a..a6c41b755f 100644 --- a/docs/contribute/pull_request.rst +++ b/docs/contribute/pull_request.rst @@ -242,7 +242,7 @@ Necessary dependencies: .. code:: bash - pip install --user pytest Cython + pip install --user pytest pytest-xdist Cython If you want to run all tests: diff --git a/python/tvm/testing/plugin.py b/python/tvm/testing/plugin.py index dfb858f1b8..2869664842 100644 --- a/python/tvm/testing/plugin.py +++ b/python/tvm/testing/plugin.py @@ -33,7 +33,6 @@ directory as the test scripts. """ import _pytest -import pytest def pytest_collection_modifyitems(config, items): @@ -44,14 +43,6 @@ def pytest_collection_modifyitems(config, items): _sort_tests(items) -def pytest_sessionfinish(session, exitstatus): - # Don't exit with an error if we select a subset of tests that doesn't - # include anything - if session.config.option.markexpr != "": - if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: - session.exitstatus = pytest.ExitCode.OK - - def _count_num_fixture_uses(items): # Helper function, counts the number of tests that use each cached # fixture. Should be called from pytest_collection_modifyitems(). diff --git a/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py b/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py index 1e209cacbc..97c9469ccf 100644 --- a/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py +++ b/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py @@ -149,7 +149,7 @@ def _dummy_result(num_samples: int = 4, max_run_sec: int = 10): @requires_xgboost def test_meta_schedule_xgb_model(): extractor = RandomFeatureExtractor() - model = XGBModel(extractor=extractor, num_warmup_samples=2) + model = XGBModel(extractor=extractor, num_warmup_samples=2, num_tuning_cores=1) update_sample_count = 10 predict_sample_count = 100 model.update( @@ -162,9 +162,10 @@ def test_meta_schedule_xgb_model(): @requires_xgboost def test_meta_schedule_xgb_model_no_feature(): - model = XGBModel(num_warmup_samples=0) + model = XGBModel(num_warmup_samples=0, num_tuning_cores=1) tune_ctx = TuneContext( FullModule, + num_threads=1, target={"kind": "llvm", "num-cores": 16}, space_generator="post-order-apply", search_strategy="evolutionary", @@ -177,7 +178,7 @@ def test_meta_schedule_xgb_model_no_feature(): @requires_xgboost def test_meta_schedule_xgb_model_reload(): extractor = RandomFeatureExtractor() - model = XGBModel(extractor=extractor, num_warmup_samples=10) + model = XGBModel(extractor=extractor, num_warmup_samples=10, num_tuning_cores=1) update_sample_count = 20 predict_sample_count = 30 model.update( @@ -221,7 +222,7 @@ def test_meta_schedule_xgb_model_reload(): @requires_xgboost def test_meta_schedule_xgb_model_reupdate(): extractor = RandomFeatureExtractor() - model = XGBModel(extractor=extractor, num_warmup_samples=2) + model = XGBModel(extractor=extractor, num_warmup_samples=2, num_tuning_cores=1) update_sample_count = 60 predict_sample_count = 100 model.update( @@ -252,7 +253,7 @@ def test_meta_schedule_xgb_model_callback_as_function(): # pylint: enable=import-outside-toplevel extractor = RandomFeatureExtractor() - model = XGBModel(extractor=extractor, num_warmup_samples=10) + model = XGBModel(extractor=extractor, num_warmup_samples=10, num_tuning_cores=1) update_sample_count = 20 predict_sample_count = 30 diff --git a/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py b/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py index 002741c6bf..6451a94ec9 100644 --- a/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py +++ b/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py @@ -103,6 +103,7 @@ def test_meta_schedule_replay_func( context = ms.TuneContext( mod=Matmul, + num_threads=1, space_generator=ms.space_generator.ScheduleFn(sch_fn=_schedule_matmul, postprocs=[]), search_strategy=TestClass(), ) diff --git a/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py b/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py index 61f5583c2a..6dc351761a 100644 --- a/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py +++ b/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py @@ -157,6 +157,7 @@ def test_meta_schedule_task_scheduler_single(): [ ms.TuneContext( MatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -183,6 +184,7 @@ def test_meta_schedule_task_scheduler_multiple(): tasks = [ ms.TuneContext( MatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -191,6 +193,7 @@ def test_meta_schedule_task_scheduler_multiple(): ), ms.TuneContext( MatmulReluModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -199,6 +202,7 @@ def test_meta_schedule_task_scheduler_multiple(): ), ms.TuneContext( BatchMatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_batch_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -255,6 +259,7 @@ def test_meta_schedule_task_scheduler_override_next_task_id_only(): # pylint: d tasks = [ ms.TuneContext( MatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -263,6 +268,7 @@ def test_meta_schedule_task_scheduler_override_next_task_id_only(): # pylint: d ), ms.TuneContext( MatmulReluModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -271,6 +277,7 @@ def test_meta_schedule_task_scheduler_override_next_task_id_only(): # pylint: d ), ms.TuneContext( BatchMatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_batch_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -310,6 +317,7 @@ def test_meta_schedule_task_scheduler_multiple_gradient_based(): tasks = [ ms.TuneContext( MatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -318,6 +326,7 @@ def test_meta_schedule_task_scheduler_multiple_gradient_based(): ), ms.TuneContext( MatmulReluModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -326,6 +335,7 @@ def test_meta_schedule_task_scheduler_multiple_gradient_based(): ), ms.TuneContext( BatchMatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_batch_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -394,6 +404,7 @@ def test_meta_schedule_task_scheduler_gradient_based_with_null_search_strategy() tasks = [ ms.TuneContext( MatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=NullSearchStrategy(rounds_with_empty_candidates=5), @@ -402,6 +413,7 @@ def test_meta_schedule_task_scheduler_gradient_based_with_null_search_strategy() ), ms.TuneContext( BatchMatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_batch_matmul, search_strategy=NullSearchStrategy(rounds_with_empty_candidates=0), @@ -410,6 +422,7 @@ def test_meta_schedule_task_scheduler_gradient_based_with_null_search_strategy() ), ms.TuneContext( MatmulReluModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), diff --git a/tests/python/s_tir/transform/test_s_tir_transform_lower_match_buffer.py b/tests/python/s_tir/transform/test_s_tir_transform_lower_match_buffer.py index a174c771a2..eaa78cf626 100644 --- a/tests/python/s_tir/transform/test_s_tir_transform_lower_match_buffer.py +++ b/tests/python/s_tir/transform/test_s_tir_transform_lower_match_buffer.py @@ -486,7 +486,7 @@ def fail_buffer_bind(a: T.handle) -> None: # well-formed checker complains about redefinition of a stride variable @T.prim_func(check_well_formed=False, s_tir=True) -def fail_match_func_param(a: T.handle, m: T.handle, n: T.handle) -> None: +def fail_match_func_param(a: T.handle, m: T.int32, n: T.int32) -> None: A = T.match_buffer(a, (8, 8)) for i, j in T.grid(8, 2): with T.sblock(): diff --git a/tests/python/testing/test_tvm_testing_features.py b/tests/python/testing/test_tvm_testing_features.py index 549aedda6a..2f4c798947 100644 --- a/tests/python/testing/test_tvm_testing_features.py +++ b/tests/python/testing/test_tvm_testing_features.py @@ -14,10 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# ruff: noqa: F401, RUF012 +# ruff: noqa: RUF012 import os -import sys import pytest diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh deleted file mode 100755 index 171ddbc2d0..0000000000 --- a/tests/scripts/setup-pytest-env.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# NOTE: allow unbound variable here -set +u - -if [[ ! -z $CI_PYTEST_ADD_OPTIONS ]]; then - export PYTEST_ADDOPTS="-s -vv $CI_PYTEST_ADD_OPTIONS $PYTEST_ADDOPTS" -else - export PYTEST_ADDOPTS="-s -vv $PYTEST_ADDOPTS" -fi -set -ux - -export TVM_PATH=`pwd` -export PYTHONPATH="${TVM_PATH}/python" - -export TVM_PYTEST_RESULT_DIR="${TVM_PATH}/build/pytest-results" -mkdir -p "${TVM_PYTEST_RESULT_DIR}" -pytest_errors=() - -# This ensures that all pytest invocations that are run through run_pytest will -# complete and errors will be reported once Bash is done executing all scripts. -function cleanup() { - set +x - if [ "${#pytest_errors[@]}" -gt 0 ]; then - echo "These pytest invocations failed, the results can be found in the Jenkins 'Tests' tab or by scrolling up through the raw logs here." - python3 ci/scripts/jenkins/pytest_wrapper.py "${pytest_errors[@]}" - exit 1 - fi - set -x -} -trap cleanup 0 - -function run_pytest() { - ffi_type="cython" - set -e - local test_suite_name="$1" - shift - extra_args=( "$@" ) - if [ -z "${ffi_type}" -o -z "${test_suite_name}" ]; then - echo "error: run_pytest called incorrectly: run_pytest ${test_suite_name}" "${extra_args[@]}" - echo "usage: run_pytest <TEST_SUITE_NAME> [pytest args...]" - exit 2 - fi - - # Allow unbound variable here. - set +u - if [[ -z "${TVM_SHARD_INDEX}" ]]; then - current_shard="no-shard" - else - current_shard="shard-${TVM_SHARD_INDEX}" - fi - set -u - - has_reruns=$(python3 -m pytest --help 2>&1 | grep 'reruns=' || true) - if [ -n "$has_reruns" ]; then - if [[ ! "${extra_args[*]}" == *"--reruns"* ]]; then - extra_args+=('--reruns=3') - fi - fi - - suite_name="${test_suite_name}-${current_shard}-${ffi_type}" - - DEFAULT_PARALLELISM=1 - - if [[ ! "${extra_args[*]}" == *" -n"* ]] && [[ ! "${extra_args[*]}" == *" -dist"* ]]; then - extra_args+=("-n=$DEFAULT_PARALLELISM") - fi - - exit_code=0 - set +e - python3 -m pytest \ - -o "junit_suite_name=${suite_name}" \ - "--junit-xml=${TVM_PYTEST_RESULT_DIR}/${suite_name}.xml" \ - "--junit-prefix=${ffi_type}" \ - "${extra_args[@]}" || exit_code=$? - # Pytest will return error code -5 if no test is collected. - if [ "$exit_code" -ne "0" ] && [ "$exit_code" -ne "5" ]; then - pytest_errors+=("${suite_name}: $@") - fi - # To avoid overwriting. - set -e -} diff --git a/tests/scripts/task_clear_pytest.sh b/tests/scripts/task_clear_pytest.sh index 0a24f62531..430425791c 100755 --- a/tests/scripts/task_clear_pytest.sh +++ b/tests/scripts/task_clear_pytest.sh @@ -16,9 +16,7 @@ # specific language governing permissions and limitations # under the License. -set -euxo pipefail +set -euo pipefail -# Ensure no stale pytest-results remain from a previous test run. -pushd build -rm -rf pytest-results -popd +# PR jobs evaluate their Jenkinsfile from the trusted base branch. Keep this +# inert entry point until the Jenkinsfile update has landed on the base branch. diff --git a/tests/scripts/task_python_docs.sh b/tests/scripts/task_python_docs.sh index 7c3122cd92..11b2501b7b 100755 --- a/tests/scripts/task_python_docs.sh +++ b/tests/scripts/task_python_docs.sh @@ -18,7 +18,7 @@ set -euxo pipefail -source tests/scripts/setup-pytest-env.sh +export PYTHONPATH="$(pwd)/python" # to avoid CI CPU thread throttling. export TVM_BIND_THREADS=0 diff --git a/tests/scripts/task_python_integration.sh b/tests/scripts/task_python_integration.sh index dcd7b8ff47..c7cf1f4329 100755 --- a/tests/scripts/task_python_integration.sh +++ b/tests/scripts/task_python_integration.sh @@ -18,7 +18,7 @@ set -euxo pipefail -source tests/scripts/setup-pytest-env.sh +export PYTHONPATH="$(pwd)/python" export LD_LIBRARY_PATH="build:${LD_LIBRARY_PATH:-}" # to avoid CI CPU thread throttling. diff --git a/tests/scripts/task_python_nightly.sh b/tests/scripts/task_python_nightly.sh deleted file mode 100755 index b480563a8c..0000000000 --- a/tests/scripts/task_python_nightly.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euxo pipefail - -source tests/scripts/setup-pytest-env.sh - -# setup tvm-ffi into python folder -uv pip install -v --target=python ./3rdparty/tvm-ffi/ - -# cleanup pycache -find . -type f -path "*.pyc" | xargs rm -f - -run_pytest python-topi-nightly tests/python/topi/nightly - -# Tensorflow device verification and network tests on nightly -export CI_ENV_NIGHTLY -python3 tests/python/relax/test_frontend_tflite.py diff --git a/tests/scripts/task_python_unittest.sh b/tests/scripts/task_python_unittest.sh index 7e09926cc0..0728743fb5 100755 --- a/tests/scripts/task_python_unittest.sh +++ b/tests/scripts/task_python_unittest.sh @@ -18,7 +18,8 @@ set -euxo pipefail -source tests/scripts/setup-pytest-env.sh +export PYTHONPATH="$(pwd)/python" +export PYTEST_ADDOPTS="-s -vv ${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f @@ -26,23 +27,20 @@ find . -type f -path "*.pyc" | xargs rm -f # setup tvm-ffi into python folder uv pip install -v --target=python ./3rdparty/tvm-ffi/ -# NOTE: also set by task_python_unittest_gpuonly.sh. -if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then - TVM_UNITTEST_TESTSUITE_NAME=python-unittest +# First run the minimal platform test. A GPU-only run can select no tests here. +if [ ! -d tests/python/all-platform-minimal-test ]; then + echo "Missing pytest target: tests/python/all-platform-minimal-test" >&2 + exit 1 fi +python3 -m pytest -n auto tests/python/all-platform-minimal-test || [ "$?" -eq 5 ] -# First run minimal test on both ctypes and cython. -run_pytest ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test tests/python/all-platform-minimal-test - -# Then run all unittests on both ctypes and cython. +# Then run all unit tests. TEST_FILES=( - "ffi" "arith" "ci" "codegen" "driver" "ir" - "meta_schedule" "runtime" "target" "te" @@ -51,6 +49,7 @@ TEST_FILES=( "s_tir/schedule" "s_tir/dlight" "s_tir/analysis" + "s_tir/meta_schedule" "s_tir/transform" "tirx-analysis" "tirx-base" @@ -62,7 +61,13 @@ TEST_FILES=( PYTEST_TARGETS=() for TEST_FILE in "${TEST_FILES[@]}"; do - PYTEST_TARGETS+=("tests/python/${TEST_FILE}") + TEST_PATH="tests/python/${TEST_FILE}" + if [ ! -e "${TEST_PATH}" ]; then + echo "Missing pytest target: ${TEST_PATH}" >&2 + exit 1 + fi + PYTEST_TARGETS+=("${TEST_PATH}") done -run_pytest "${TVM_UNITTEST_TESTSUITE_NAME}" "${PYTEST_TARGETS[@]}" +# Do not mask pytest's exit 5: an unexpectedly empty broad suite must fail CI. +python3 -m pytest -n auto --dist=loadgroup "${PYTEST_TARGETS[@]}" diff --git a/tests/scripts/task_python_unittest_gpuonly.sh b/tests/scripts/task_python_unittest_gpuonly.sh index be3ea07947..8194b23ac4 100755 --- a/tests/scripts/task_python_unittest_gpuonly.sh +++ b/tests/scripts/task_python_unittest_gpuonly.sh @@ -24,15 +24,18 @@ export PYTEST_ADDOPTS="-m gpu ${PYTEST_ADDOPTS:-}" # Test most of the enabled runtimes here. # TODO: disabled opencl tests due to segmentation fault. export TVM_TEST_TARGETS='cuda;metal;rocm;nvptx' -export TVM_UNITTEST_TESTSUITE_NAME=python-unittest-gpu ./tests/scripts/task_python_unittest.sh # Kept separate to avoid increasing time needed to run CI, testing # only minimal functionality of Vulkan runtime. export TVM_TEST_TARGETS='{"kind":"vulkan","from_device":0}' -export TVM_UNITTEST_TESTSUITE_NAME=python-codegen-vulkan -source tests/scripts/setup-pytest-env.sh +export PYTHONPATH="$(pwd)/python" +export PYTEST_ADDOPTS="-s -vv ${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" -run_pytest ${TVM_UNITTEST_TESTSUITE_NAME}-1 tests/python/codegen/test_target_codegen_vulkan.py +if [ ! -f tests/python/codegen/test_target_codegen_vulkan.py ]; then + echo "Missing pytest target: tests/python/codegen/test_target_codegen_vulkan.py" >&2 + exit 1 +fi +python3 -m pytest -n auto tests/python/codegen/test_target_codegen_vulkan.py
