This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch tvm-simplify-jenkins-pytest-sharding in repository https://gitbox.apache.org/repos/asf/tvm.git
commit c73279e84b444ff67644a23f05e2758f9fc2c9ec Author: Tianqi Chen <[email protected]> AuthorDate: Sun Jul 5 01:43:50 2026 +0000 [CI] Simplify Jenkins pytest execution Remove external test sharding, obsolete pytest wrapper machinery, JUnit reporting and publication, and the skipped-test XML consumer. Repair broad-suite targets and empty-selection failure propagation, while bounding active nested MetaSchedule test parallelism without blanket serialization. --- ci/jenkins/generated/arm_jenkinsfile.groovy | 11 +- ci/jenkins/generated/cpu_jenkinsfile.groovy | 98 +------- ci/jenkins/generated/docker_jenkinsfile.groovy | 11 +- ci/jenkins/generated/gpu_jenkinsfile.groovy | 129 +--------- ci/jenkins/generated/wasm_jenkinsfile.groovy | 12 +- ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 | 4 +- ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 | 10 +- ci/jenkins/templates/utils/Build.groovy.j2 | 7 - ci/jenkins/templates/utils/base.groovy.j2 | 2 +- ci/jenkins/templates/utils/macros.j2 | 30 +-- ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 | 1 - ci/scripts/github/__init__.py | 2 +- ci/scripts/github/github_pr_comment.py | 4 - ci/scripts/github/github_skipped_tests_comment.py | 263 ------------------- ci/scripts/jenkins/pytest_ids.py | 43 ---- ci/scripts/jenkins/pytest_wrapper.py | 137 ---------- conftest.py | 54 ---- docker/install/ubuntu_install_python_package.sh | 2 - python/tvm/testing/plugin.py | 9 - tests/python/ci/test_ci.py | 280 +-------------------- .../meta_schedule/test_meta_schedule_cost_model.py | 11 +- .../test_meta_schedule_search_strategy.py | 1 + .../test_meta_schedule_task_scheduler.py | 13 + tests/python/testing/test_tvm_testing_features.py | 42 +++- tests/scripts/task_clear_pytest.sh | 24 -- tests/scripts/task_python_nightly.sh | 39 --- tests/scripts/task_python_unittest.sh | 34 ++- tests/scripts/task_python_unittest_gpuonly.sh | 14 +- 28 files changed, 117 insertions(+), 1170 deletions(-) diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy index d6826878bf..3442caded7 100644 --- a/ci/jenkins/generated/arm_jenkinsfile.groovy +++ b/ci/jenkins/generated/arm_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.520193 +// Generated at 2026-07-05T01:05:57.719405 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 @@ -404,13 +404,6 @@ def prepare(node_type) { } } } -def ci_setup(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh", - label: 'Clean up old workspace', - ) -} - def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy index 879cd8b79f..0d3e408235 100644 --- a/ci/jenkins/generated/cpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.505504 +// Generated at 2026-07-05T01:05:57.741181 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 @@ -404,13 +404,6 @@ def prepare(node_type) { } } } -def ci_setup(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh", - label: 'Clean up old workspace', - ) -} - def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", @@ -510,52 +503,7 @@ build() - -def shard_run_unittest_CPU_1_of_2(node_type) { - echo 'Begin running on node_type ' + node_type - if (!skip_ci && is_docs_only_build != 1) { - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") { - // NOTE: if exception happens, it will be caught outside - init_git() - docker_init(ci_cpu) - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=cpu', - 'TEST_STEP_NAME=unittest: CPU', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=0', - "SKIP_SLOW_TESTS=${skip_slow_tests}"], { - sh( - script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu", - label: 'Download artifacts from S3', - ) - - ci_setup(ci_cpu) - cpp_unittest(ci_cpu) - python_unittest(ci_cpu) - }) - } - // only run upload if things are successful - try { - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU --items build/pytest-results", - label: 'Upload JUnits to S3', - ) - - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } - } - } - echo 'End running on node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('unittest: CPU 1 of 2') - } -} - -def shard_run_unittest_CPU_2_of_2(node_type) { +def run_unittest_CPU(node_type) { echo 'Begin running on node_type ' + node_type if (!skip_ci && is_docs_only_build != 1) { node(node_type) { @@ -567,35 +515,21 @@ def shard_run_unittest_CPU_2_of_2(node_type) { withEnv([ 'PLATFORM=cpu', 'TEST_STEP_NAME=unittest: CPU', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh( script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu", label: 'Download artifacts from S3', ) - ci_setup(ci_cpu) cpp_unittest(ci_cpu) python_unittest(ci_cpu) }) } - // only run upload if things are successful - try { - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU --items build/pytest-results", - label: 'Upload JUnits to S3', - ) - - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } } } echo 'End running on node_type ' + node_type } else { - Utils.markStageSkippedForConditional('unittest: CPU 2 of 2') + Utils.markStageSkippedForConditional('unittest: CPU') } } @@ -606,27 +540,9 @@ def test() { SKIP_SLOW_TESTS = "${skip_slow_tests}" } parallel( - 'unittest: CPU 1 of 2': { - try { - shard_run_unittest_CPU_1_of_2('CPU-SMALL-SPOT') - } catch (Throwable ex) { - echo 'Exception during SPOT run ' + ex.toString() - if (is_last_build()) { - // retry if at last build - // mark the current stage as success - // and try again via on demand node - echo 'Retry on-demand given it is last build' - currentBuild.result = 'SUCCESS' - shard_run_unittest_CPU_1_of_2('CPU-SMALL') - } else { - echo 'Exit since it is not last build' - throw ex - } - } - }, - 'unittest: CPU 2 of 2': { + 'unittest: CPU': { try { - shard_run_unittest_CPU_2_of_2('CPU-SMALL-SPOT') + run_unittest_CPU('CPU-SMALL-SPOT') } catch (Throwable ex) { echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { @@ -635,7 +551,7 @@ def test() { // and try again via on demand node echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' - shard_run_unittest_CPU_2_of_2('CPU-SMALL') + run_unittest_CPU('CPU-SMALL') } else { echo 'Exit since it is not last build' throw ex diff --git a/ci/jenkins/generated/docker_jenkinsfile.groovy b/ci/jenkins/generated/docker_jenkinsfile.groovy index f5409e23aa..8ffe78942a 100644 --- a/ci/jenkins/generated/docker_jenkinsfile.groovy +++ b/ci/jenkins/generated/docker_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.487314 +// Generated at 2026-07-05T01:05:57.759905 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 @@ -404,13 +404,6 @@ def prepare(node_type) { } } } -def ci_setup(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh", - label: 'Clean up old workspace', - ) -} - def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy index 28c9a6040e..792b7eadc4 100644 --- a/ci/jenkins/generated/gpu_jenkinsfile.groovy +++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.533751 +// Generated at 2026-07-05T01:05:57.780838 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 @@ -404,13 +404,6 @@ def prepare(node_type) { } } } -def ci_setup(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh", - label: 'Clean up old workspace', - ) -} - def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", @@ -515,8 +508,7 @@ def build() { build() - -def shard_run_unittest_GPU_1_of_2(node_type) { +def run_unittest_GPU(node_type) { echo 'Begin running on node_type ' + node_type if (!skip_ci && is_docs_only_build != 1) { node(node_type) { @@ -528,65 +520,12 @@ def shard_run_unittest_GPU_1_of_2(node_type) { withEnv([ 'PLATFORM=gpu', 'TEST_STEP_NAME=unittest: GPU', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh( script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu", label: 'Download artifacts from S3', ) - ci_setup(ci_gpu) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", - label: 'Run Python GPU unit tests', - ) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh", - label: 'Run Python GPU integration tests', - ) - }) - } - // only run upload if things are successful - try { - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items build/pytest-results", - label: 'Upload JUnits to S3', - ) - - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } - } - } - echo 'End running on node_type ' + node_type - } else { - Utils.markStageSkippedForConditional('unittest: GPU 1 of 2') - } -} - -def shard_run_unittest_GPU_2_of_2(node_type) { - echo 'Begin running on node_type ' + node_type - if (!skip_ci && is_docs_only_build != 1) { - node(node_type) { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { - // NOTE: if exception happens, it will be caught outside - init_git() - docker_init(ci_gpu) - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=gpu', - 'TEST_STEP_NAME=unittest: GPU', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=1', - "SKIP_SLOW_TESTS=${skip_slow_tests}"], { - sh( - script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu", - label: 'Download artifacts from S3', - ) - - ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh", label: 'Run Java unit tests', @@ -601,29 +540,17 @@ def shard_run_unittest_GPU_2_of_2(node_type) { ) }) } - // only run upload if things are successful - try { - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items build/pytest-results", - label: 'Upload JUnits to S3', - ) - - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } } } echo 'End running on node_type ' + node_type } else { - Utils.markStageSkippedForConditional('unittest: GPU 2 of 2') + Utils.markStageSkippedForConditional('unittest: GPU') } } - -def shard_run_docs_GPU_1_of_1(node_type) { +def run_docs_GPU(node_type) { echo 'Begin running on node_type ' + node_type if (!skip_ci) { node(node_type) { @@ -635,15 +562,12 @@ def shard_run_docs_GPU_1_of_1(node_type) { withEnv([ 'PLATFORM=gpu', 'TEST_STEP_NAME=docs: GPU', - 'TVM_NUM_SHARDS=1', - 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh( script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu", label: 'Download artifacts from S3', ) - ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh", label: 'Build docs', @@ -659,22 +583,11 @@ def shard_run_docs_GPU_1_of_1(node_type) { ) }) } - // only run upload if things are successful - try { - sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/docs_GPU --items build/pytest-results", - label: 'Upload JUnits to S3', - ) - - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } } } echo 'End running on node_type ' + node_type } else { - Utils.markStageSkippedForConditional('docs: GPU 1 of 1') + Utils.markStageSkippedForConditional('docs: GPU') } } @@ -686,27 +599,9 @@ def test() { SKIP_SLOW_TESTS = "${skip_slow_tests}" } parallel( - 'unittest: GPU 1 of 2': { - try { - shard_run_unittest_GPU_1_of_2('GPU-SPOT') - } catch (Throwable ex) { - echo 'Exception during SPOT run ' + ex.toString() - if (is_last_build()) { - // retry if at last build - // mark the current stage as success - // and try again via on demand node - echo 'Retry on-demand given it is last build' - currentBuild.result = 'SUCCESS' - shard_run_unittest_GPU_1_of_2('GPU') - } else { - echo 'Exit since it is not last build' - throw ex - } - } - }, - 'unittest: GPU 2 of 2': { + 'unittest: GPU': { try { - shard_run_unittest_GPU_2_of_2('GPU-SPOT') + run_unittest_GPU('GPU-SPOT') } catch (Throwable ex) { echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { @@ -715,16 +610,16 @@ def test() { // and try again via on demand node echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' - shard_run_unittest_GPU_2_of_2('GPU') + run_unittest_GPU('GPU') } else { echo 'Exit since it is not last build' throw ex } } }, - 'docs: GPU 1 of 1': { + 'docs: GPU': { try { - shard_run_docs_GPU_1_of_1('GPU-SPOT') + run_docs_GPU('GPU-SPOT') } catch (Throwable ex) { echo 'Exception during SPOT run ' + ex.toString() if (is_last_build()) { @@ -733,7 +628,7 @@ def test() { // and try again via on demand node echo 'Retry on-demand given it is last build' currentBuild.result = 'SUCCESS' - shard_run_docs_GPU_1_of_1('GPU') + run_docs_GPU('GPU') } else { echo 'Exit since it is not last build' throw ex diff --git a/ci/jenkins/generated/wasm_jenkinsfile.groovy b/ci/jenkins/generated/wasm_jenkinsfile.groovy index ab64444e4a..7421fa86ca 100644 --- a/ci/jenkins/generated/wasm_jenkinsfile.groovy +++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy @@ -60,7 +60,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2026-06-23T03:28:14.550555 +// Generated at 2026-07-05T01:05:57.802104 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // These are set at runtime from data in ci/jenkins/docker-images.yml, update @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 @@ -404,13 +404,6 @@ def prepare(node_type) { } } } -def ci_setup(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh", - label: 'Clean up old workspace', - ) -} - def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", @@ -471,7 +464,6 @@ def run_build(node_type) { cmake_build(ci_wasm, 'build') make_cpp_tests(ci_wasm, 'build') cpp_unittest(ci_wasm) - ci_setup(ci_wasm) sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh", label: 'Run WASM lint and tests', diff --git a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 index d2e479d5e8..62b7c3dfea 100644 --- a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 +++ b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 @@ -37,16 +37,14 @@ {% set test_method_names = [] %} -{% call(shard_index, num_shards) m.sharded_test_step( +{% call m.test_step( name="unittest: CPU", ws="tvm/ut-python-cpu", platform="cpu", - num_shards=2, docker_image="ci_cpu", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='cpu') }} - ci_setup(ci_cpu) cpp_unittest(ci_cpu) python_unittest(ci_cpu) {% endcall %} diff --git a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 index 7ab5256419..3a13dfe7f3 100644 --- a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 +++ b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 @@ -38,22 +38,18 @@ {% set test_method_names = [] %} -{% call(shard_index, num_shards) m.sharded_test_step( +{% call m.test_step( name="unittest: GPU", - num_shards=2, ws="tvm/ut-python-gpu", platform="gpu", docker_image="ci_gpu", test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='gpu') }} - ci_setup(ci_gpu) - {% if shard_index == 2 or num_shards < 2 %} sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh", label: 'Run Java unit tests', ) - {% endif %} sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", label: 'Run Python GPU unit tests', @@ -65,9 +61,8 @@ {% endcall %} -{% call(shard_index, num_shards) m.sharded_test_step( +{% call m.test_step( name="docs: GPU", - num_shards=1, ws="tvm/docs-python-gpu", platform="gpu", docker_image="ci_gpu", @@ -75,7 +70,6 @@ test_method_names=test_method_names, ) %} {{ m.download_artifacts(tag='gpu') }} - ci_setup(ci_gpu) sh ( script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh", label: 'Build docs', diff --git a/ci/jenkins/templates/utils/Build.groovy.j2 b/ci/jenkins/templates/utils/Build.groovy.j2 index c20a1c5c43..42bd4a991f 100644 --- a/ci/jenkins/templates/utils/Build.groovy.j2 +++ b/ci/jenkins/templates/utils/Build.groovy.j2 @@ -1,10 +1,3 @@ -def ci_setup(image) { - sh ( - script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh", - label: 'Clean up old workspace', - ) -} - def python_unittest(image) { sh ( script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh", diff --git a/ci/jenkins/templates/utils/base.groovy.j2 b/ci/jenkins/templates/utils/base.groovy.j2 index 813ceee886..a8fda460b7 100644 --- a/ci/jenkins/templates/utils/base.groovy.j2 +++ b/ci/jenkins/templates/utils/base.groovy.j2 @@ -82,7 +82,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' +docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 diff --git a/ci/jenkins/templates/utils/macros.j2 b/ci/jenkins/templates/utils/macros.j2 index b1bd3679ac..959d91725f 100644 --- a/ci/jenkins/templates/utils/macros.j2 +++ b/ci/jenkins/templates/utils/macros.j2 @@ -19,18 +19,8 @@ "workspace/exec_${env.EXECUTOR_NUMBER}/{{ folder }}" {%- endmacro -%} -{% macro junit_to_s3(test_dir_name) %} -sh( - script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/{{ test_dir_name }} --items build/pytest-results", - label: 'Upload JUnits to S3', - ) -{% endmacro %} - -{% macro sharded_test_step(name, num_shards, ws, docker_image, platform, test_method_names, condition="!skip_ci && is_docs_only_build != 1") %} - -{% for shard_index in range(1, num_shards + 1) %} -{% set method_name = "shard_run_" + name.replace(":", "").replace(" ", "-").replace("-", "_") + "_" + shard_index|string + "_of_" + num_shards|string %} -{% set test_dir_name = name.replace(":", "").replace(" ", "-").replace("-", "_")|string %} +{% macro test_step(name, ws, docker_image, platform, test_method_names, condition="!skip_ci && is_docs_only_build != 1") %} +{% set method_name = "run_" + name.replace(":", "").replace(" ", "-").replace("-", "_") %} def {{ method_name }}(node_type) { echo 'Begin running on node_type ' + node_type if ({{ condition }}) { @@ -43,29 +33,19 @@ def {{ method_name }}(node_type) { withEnv([ 'PLATFORM={{ platform }}', 'TEST_STEP_NAME={{ name }}', - 'TVM_NUM_SHARDS={{ num_shards }}', - 'TVM_SHARD_INDEX={{ shard_index - 1 }}', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { - {{ caller(shard_index, num_shards) | trim | indent(width=12) }} + {{ caller() | trim | indent(width=12) }} }) } - // only run upload if things are successful - try { - {{ junit_to_s3(test_dir_name) }} - junit 'build/pytest-results/*.xml' - } catch (Exception e) { - echo 'Exception during JUnit upload: ' + e.toString() - } } } echo 'End running on node_type ' + node_type } else { - Utils.markStageSkippedForConditional('{{ name }} {{ shard_index }} of {{ num_shards }}') + Utils.markStageSkippedForConditional('{{ name }}') } } -{% set _ = test_method_names.append((name + " " + shard_index|string + " of " + num_shards|string, method_name)) %} +{% set _ = test_method_names.append((name, method_name)) %} -{% endfor %} {% endmacro %} {% macro invoke_build(name, condition, node, docker_image, ws, platform) %} diff --git a/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 b/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 index 0d119f043e..8e7d909bd1 100644 --- a/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 +++ b/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 @@ -32,7 +32,6 @@ cmake_build(ci_wasm, 'build') make_cpp_tests(ci_wasm, 'build') cpp_unittest(ci_wasm) - ci_setup(ci_wasm) sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh", label: 'Run WASM lint and tests', diff --git a/ci/scripts/github/__init__.py b/ci/scripts/github/__init__.py index a6226fec29..0e55424eee 100644 --- a/ci/scripts/github/__init__.py +++ b/ci/scripts/github/__init__.py @@ -17,4 +17,4 @@ # under the License. """Package to enable testing of GitHub scripts""" -from . import github_skipped_tests_comment, github_pr_comment, github_tag_teams, github_docs_comment +from . import github_docs_comment, github_pr_comment, github_tag_teams diff --git a/ci/scripts/github/github_pr_comment.py b/ci/scripts/github/github_pr_comment.py index 2d523b21b3..81347659d9 100755 --- a/ci/scripts/github/github_pr_comment.py +++ b/ci/scripts/github/github_pr_comment.py @@ -30,7 +30,6 @@ from cmd_utils import init_log from git_utils import DRY_RUN, GitHubRepo, git, parse_remote from github_commenter import BotCommentBuilder from github_docs_comment import get_doc_url -from github_skipped_tests_comment import get_skipped_tests_comment from github_tag_teams import get_tags PR_QUERY = """ @@ -132,17 +131,14 @@ if __name__ == "__main__": if args.test_comments is not None: test_comments = json.loads(args.test_comments) - skipped_tests = test_comments["skipped-tests"] ccs = test_comments["ccs"] docs_info = test_comments["docs"] else: - skipped_tests = get_skipped_tests_comment(pr_data, github=github) ccs = get_tags(pr_data, github, team_issue=10317) docs_info = get_doc_url(pr_data) items = { "ccs": ccs, - "skipped-tests": skipped_tests, "docs": docs_info, } commenter.post_items(items=items.items()) diff --git a/ci/scripts/github/github_skipped_tests_comment.py b/ci/scripts/github/github_skipped_tests_comment.py deleted file mode 100755 index 10789f7561..0000000000 --- a/ci/scripts/github/github_skipped_tests_comment.py +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ruff: noqa: E501 -import json -import logging -import os -import subprocess -from pathlib import Path -from typing import Any -from xml.etree import ElementTree - - -def run_subprocess(command): - logging.info(f"Running command {command}") - proc = subprocess.run(command, shell=True, stdout=subprocess.PIPE, encoding="utf-8") - if proc.returncode != 0: - raise RuntimeError(f"Command failed {command}:\nstdout:\n{proc.stdout}") - return proc - - -def retrieve_test_report(s3_url, target_dir): - command = f"aws --region us-west-2 s3 cp {s3_url} {target_dir} --recursive --no-sign-request" - run_subprocess(command) - - -def get_common_commit_sha(): - command = "git merge-base origin/main HEAD" - proc = run_subprocess(command) - return proc.stdout.strip() - - -def get_main_jenkins_build_number(github, common_commit): - json = github.get(f"commits/{common_commit}/status") - for status in reversed(json["statuses"]): - if status["context"] != "tvm-ci/branch": - continue - state = status["state"] - target_url = str(status["target_url"]) - build_number = ( - target_url[target_url.find("job/main") : len(target_url)] - .strip("job/main/") - .strip("/display/redirect") - ) - assert build_number.isdigit() - return {"build_number": build_number, "state": state} - raise RuntimeError(f"Failed to find main build number for commit {common_commit}") - - -def retrieve_test_reports( - common_main_build, pr_number, build_number, s3_prefix, pr_test_report_dir, main_test_report_dir -): - cur_build_s3_link = f"s3://{s3_prefix}/tvm/PR-{pr_number!s}/{build_number!s}/pytest-results" - retrieve_test_report(cur_build_s3_link, pr_test_report_dir) - - common_build_s3_link = f"s3://{s3_prefix}/tvm/main/{common_main_build}/pytest-results" - retrieve_test_report(common_build_s3_link, main_test_report_dir) - - -def get_pr_and_build_numbers(target_url): - target_url = target_url[target_url.find("PR-") : len(target_url)] - split = target_url.split("/") - pr_number = split[0].strip("PR-") - build_number = split[1] - return {"pr_number": pr_number, "build_number": build_number} - - -def build_test_set(directory): - directory = Path(directory) - subdir_to_skipped = {} - subdirs = [ - item for item in os.listdir(directory) if os.path.isdir(os.path.join(directory, item)) - ] - for subdir in subdirs: - subdir_to_skipped[subdir] = set() - for root, _, files in os.walk(directory / subdir): - for file in files: - test_report = ElementTree.parse(Path(root) / file) - for testcase in test_report.iter("testcase"): - skipped = testcase.find("skipped") - if skipped is not None: - key = testcase.attrib["classname"] + "#" + testcase.attrib["name"] - subdir_to_skipped[subdir].add(key) - return subdir_to_skipped - - -def to_node_name(dir_name: str): - return dir_name.replace("_", ": ", 1) - - -def build_diff_comment_with_main( - common_commit_sha, - skipped_list, - commit_sha, -): - if len(skipped_list) == 0: - return f"No diff in skipped tests with main found in this branch for commit {commit_sha}.\n" - - text = ( - f"The list below shows tests that ran in main {common_commit_sha} but were " - f"skipped in the CI build of {commit_sha}:\n" - f"```\n" - ) - for skip in skipped_list: - text += skip + "\n" - text += "```\n" - return text - - -def build_comment( - common_commit_sha, - common_main_build, - skipped_list, - additional_skipped_list, - pr_number, - build_number, - commit_sha, - jenkins_prefix, -): - if common_main_build["state"] != "success": - return f"Unable to run tests bot because main failed to pass CI at {common_commit_sha}." - - text = build_diff_comment_with_main(common_commit_sha, skipped_list, commit_sha) - - if len(additional_skipped_list) != 0: - text += "\n" - text += ( - "Additional tests that were skipped in the CI build and present in the [`required_tests_to_run`]" - "(https://github.com/apache/tvm/blob/main/ci/scripts/github/required_tests_to_run.json) file:" - "\n```\n" - ) - for skip in additional_skipped_list: - text += skip + "\n" - text += "```\n" - - text += ( - f"A detailed report of ran tests is [here](https://{jenkins_prefix}/job/tvm/job/PR-{pr_number!s}"; - f"/{build_number!s}/testReport/)." - ) - return text - - -def find_target_url(pr_head: dict[str, Any]): - for status in pr_head["statusCheckRollup"]["contexts"]["nodes"]: - if status.get("context", "") == "tvm-ci/pr-head": - return status["targetUrl"] - - raise RuntimeError(f"Unable to find tvm-ci/pr-head status in {pr_head}") - - -def get_skipped_tests_comment( - pr: dict[str, Any], - github, - s3_prefix: str = "tvm-jenkins-artifacts-prod", - jenkins_prefix: str = "ci.tlcpack.ai", - pr_test_report_dir: str = "pr-reports", - main_test_report_dir: str = "main-reports", - common_commit_sha: str | None = None, - common_main_build: dict[str, Any] | None = None, - additional_tests_to_check_file: str = "required_tests_to_run.json", -) -> str: - pr_head = pr["commits"]["nodes"][0]["commit"] - target_url = find_target_url(pr_head) - pr_and_build = get_pr_and_build_numbers(target_url) - logging.info(f"Getting comment for {pr_head} with target {target_url}") - - commit_sha = pr_head["oid"] - - is_dry_run = common_commit_sha is not None - - if not is_dry_run: - logging.info("Fetching common commit sha and build info") - common_commit_sha = get_common_commit_sha() - common_main_build = get_main_jenkins_build_number(github, common_commit_sha) - - retrieve_test_reports( - common_main_build=common_main_build["build_number"], - pr_number=pr_and_build["pr_number"], - build_number=pr_and_build["build_number"], - s3_prefix=s3_prefix, - main_test_report_dir=main_test_report_dir, - pr_test_report_dir=pr_test_report_dir, - ) - else: - logging.info("Dry run, expecting PR and main reports on disk") - - main_tests = build_test_set(main_test_report_dir) - build_tests = build_test_set(pr_test_report_dir) - - skipped_list = [] - for subdir, skipped_set in build_tests.items(): - skipped_main = main_tests[subdir] - if skipped_main is None: - logging.warning(f"Could not find directory {subdir} in main.") - continue - - diff_set = skipped_set - skipped_main - if len(diff_set) != 0: - for test in diff_set: - skipped_list.append(f"{to_node_name(subdir)} -> {test}") - - # Sort the list to maintain an order in the output. Helps when validating the output in tests. - skipped_list.sort() - - if len(skipped_list) == 0: - logging.info("No skipped tests found.") - - if not is_dry_run: - current_file = Path(__file__).resolve() - additional_tests_to_check_file = Path(current_file).parent / "required_tests_to_run.json" - - logging.info( - f"Checking additional tests in file {additional_tests_to_check_file} are not skipped." - ) - try: - with open(additional_tests_to_check_file) as f: - additional_tests_to_check = json.load(f) - except OSError: - logging.info( - f"Failed to read additional tests from file: {additional_tests_to_check_file}." - ) - additional_tests_to_check = {} - - # Assert that tests present in "required_tests_to_run.json" are not skipped. - additional_skipped_tests = [] - for subdir, test_set in additional_tests_to_check.items(): - if subdir not in build_tests.keys(): - logging.warning(f"Could not find directory {subdir} in the build test set.") - continue - - for test in test_set: - if test in build_tests[subdir]: - additional_skipped_tests.append(f"{to_node_name(subdir)} -> {test}") - - if len(additional_skipped_tests) == 0: - logging.info("No skipped tests found in the additional list.") - - body = build_comment( - common_commit_sha, - common_main_build, - skipped_list, - additional_skipped_tests, - pr_and_build["pr_number"], - pr_and_build["build_number"], - commit_sha, - jenkins_prefix, - ) - - return body diff --git a/ci/scripts/jenkins/pytest_ids.py b/ci/scripts/jenkins/pytest_ids.py deleted file mode 100755 index 548069e855..0000000000 --- a/ci/scripts/jenkins/pytest_ids.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -import argparse -import io -from contextlib import redirect_stdout - -import pytest - - -class NodeidsCollector: - def pytest_collection_modifyitems(self, items): - self.nodeids = [item.nodeid for item in items] - - -def main(folder): - collector = NodeidsCollector() - f = io.StringIO() - with redirect_stdout(f): - pytest.main(["-qq", "--collect-only", folder], plugins=[collector]) - for nodeid in collector.nodeids: - print(nodeid) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="List pytest nodeids for a folder") - parser.add_argument("--folder", required=True, help="test folder to inspect") - args = parser.parse_args() - main(args.folder) diff --git a/ci/scripts/jenkins/pytest_wrapper.py b/ci/scripts/jenkins/pytest_wrapper.py deleted file mode 100755 index a667810293..0000000000 --- a/ci/scripts/jenkins/pytest_wrapper.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ruff: noqa: E501 -import argparse -import logging -import os -import textwrap -import urllib.parse -from pathlib import Path - -import junitparser -from cmd_utils import init_log - -REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent - - -def lstrip(s: str, prefix: str) -> str: - if s.startswith(prefix): - s = s[len(prefix) :] - return s - - -def classname_to_file(classname: str) -> str: - classname = lstrip(classname, "cython.") - classname = lstrip(classname, "ctypes.") - return classname.replace(".", "/") + ".py" - - -def failed_test_ids() -> list[str]: - FAILURE_TYPES = (junitparser.Failure, junitparser.Error) - junit_dir = REPO_ROOT / "build" / "pytest-results" - failed_node_ids = [] - for junit in junit_dir.glob("*.xml"): - xml = junitparser.JUnitXml.fromfile(str(junit)) - for suite in xml: - # handle suites - for case in suite: - if case.result is None: - logging.warn(f"Incorrectly formatted JUnit found, result was None on {case}") - continue - - if len(case.result) > 0 and isinstance(case.result[0], FAILURE_TYPES): - node_id = classname_to_file(case.classname) + "::" + case.name - failed_node_ids.append(node_id) - - return list(set(failed_node_ids)) - - -def repro_command(build_type: str, failed_node_ids: list[str]) -> str | None: - """ - Parse available JUnit XML files and output a command that users can run to - reproduce CI failures locally - """ - test_args = [f"--tests {node_id}" for node_id in failed_node_ids] - test_args_str = " ".join(test_args) - return f"python3 tests/scripts/ci.py {build_type} {test_args_str}" - - -def make_issue_url(failed_node_ids: list[str]) -> str: - names = [f"`{node_id}`" for node_id in failed_node_ids] - run_url = os.getenv("RUN_DISPLAY_URL", "<insert run URL>") - test_bullets = [f" - `{node_id}`" for node_id in failed_node_ids] - params = { - "labels": "test: flaky", - "title": "[Flaky Test] " + ", ".join(names), - "body": textwrap.dedent( - """ - These tests were found to be flaky (intermittently failing on `main` or failed in a PR with unrelated changes). See [the docs](https://github.com/apache/tvm/blob/main/docs/contribute/ci.rst#handling-flaky-failures) for details. - - ### Tests(s)\n - """ - ) - + "\n".join(test_bullets) - + f"\n\n### Jenkins Links\n\n - {run_url}", - } - return "https://github.com/apache/tvm/issues/new?"; + urllib.parse.urlencode(params) - - -def show_failure_help(failed_suites: list[str]) -> None: - failed_node_ids = failed_test_ids() - - if len(failed_node_ids) == 0: - return - - build_type = os.getenv("PLATFORM") - - if build_type is None: - raise RuntimeError("build type was None, cannot show command") - - repro = repro_command(build_type=build_type, failed_node_ids=failed_node_ids) - if repro is None: - print("No test failures detected") - return - - print(f"Report flaky test shortcut: {make_issue_url(failed_node_ids)}") - print("=============================== PYTEST FAILURES ================================") - print( - "These pytest suites failed to execute. The results can be found in the " - "Jenkins 'Tests' tab or by scrolling up through the raw logs here. " - "If there is no test listed below, the failure likely came from a segmentation " - "fault which you can find in the logs above.\n" - ) - if failed_suites is not None and len(failed_suites) > 0: - print("\n".join([f" - {suite}" for suite in failed_suites])) - print("") - - print("You can reproduce these specific failures locally with this command:\n") - print(textwrap.indent(repro, prefix=" ")) - print("") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Print information about a failed pytest run") - args, other = parser.parse_known_args() - init_log() - - try: - show_failure_help(failed_suites=other) - except Exception as e: - # This script shouldn't ever introduce failures since it's just there to - # add extra information, so ignore any errors - logging.exception(e) diff --git a/conftest.py b/conftest.py index dd32937620..dd294ea548 100644 --- a/conftest.py +++ b/conftest.py @@ -14,8 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# ruff: noqa: E501 -import hashlib import os import sys from pathlib import Path @@ -25,58 +23,6 @@ IS_IN_CI = os.getenv("CI", "") == "true" REPO_ROOT = Path(__file__).resolve().parent -# These are long running tests (manually curated and extracted from CI logs) -# that should be allocated to test shards in a round-robin fashion. These are -# taken from the 20 (arbitrary number) of tests as from -# https://ci.tlcpack.ai/job/tvm/job/main/2907/testReport -_slowest_tests = [] -HARDCODED_ALLOCATIONS = {} -for idx, test in enumerate(_slowest_tests): - HARDCODED_ALLOCATIONS[test] = idx - -# These rely on running on the same node to pass successfully -FIXED_ALLOCATION_PREFIXES = { - "tests/python/testing/test_tvm_testing_features.py": 0, -} - - -def find_shard_index(nodeid: str, num_shards: int) -> int: - """ - Return the index of the shard that should run this test - """ - for prefix, target_shard_idx in FIXED_ALLOCATION_PREFIXES.items(): - if nodeid.startswith(prefix): - if target_shard_idx >= num_shards: - raise RuntimeError( - f"Cannot collect sharded tests, {nodeid} has hardcoded shard index {target_shard_idx} among only {num_shards} shards" - ) - return target_shard_idx - - if nodeid in HARDCODED_ALLOCATIONS: - hash = HARDCODED_ALLOCATIONS[nodeid] - else: - hash = hashlib.md5(nodeid.encode()) - hash = int(hash.hexdigest(), 16) - - return hash % num_shards - - -def pytest_collection_modifyitems(config, items): - if not all(k in os.environ for k in ["CI", "TVM_NUM_SHARDS", "TVM_SHARD_INDEX"]): - # Only apportion tests if in CI and in a job that is set up for it - return - - num_shards = int(os.environ["TVM_NUM_SHARDS"]) - shard_index = int(os.environ["TVM_SHARD_INDEX"]) - - print(f"Marking tests for shard {shard_index} of {num_shards}") - items_copy = list(items) - for item in items_copy: - item_shard_index = find_shard_index(item.nodeid, num_shards=num_shards) - if item_shard_index != shard_index: - items.remove(item) - - def pytest_sessionstart(): if IS_IN_CI: hook_script_dir = REPO_ROOT / "tests" / "scripts" / "request_hook" diff --git a/docker/install/ubuntu_install_python_package.sh b/docker/install/ubuntu_install_python_package.sh index d07f19ed3c..4445dfd83e 100755 --- a/docker/install/ubuntu_install_python_package.sh +++ b/docker/install/ubuntu_install_python_package.sh @@ -32,13 +32,11 @@ uv pip install --upgrade \ Pillow==12.1.1 \ "psutil~=7.0" \ "pytest~=8.3" \ - "pytest-profiling~=1.8" \ "pytest-xdist~=3.6" \ pytest-rerunfailures==16.1 \ "requests~=2.32" \ "scipy~=1.13" \ "Jinja2~=3.1" \ - junitparser==4.0.2 \ "six~=1.17" \ "tornado~=6.4" \ "ml_dtypes~=0.5" \ diff --git a/python/tvm/testing/plugin.py b/python/tvm/testing/plugin.py index dfb858f1b8..2869664842 100644 --- a/python/tvm/testing/plugin.py +++ b/python/tvm/testing/plugin.py @@ -33,7 +33,6 @@ directory as the test scripts. """ import _pytest -import pytest def pytest_collection_modifyitems(config, items): @@ -44,14 +43,6 @@ def pytest_collection_modifyitems(config, items): _sort_tests(items) -def pytest_sessionfinish(session, exitstatus): - # Don't exit with an error if we select a subset of tests that doesn't - # include anything - if session.config.option.markexpr != "": - if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: - session.exitstatus = pytest.ExitCode.OK - - def _count_num_fixture_uses(items): # Helper function, counts the number of tests that use each cached # fixture. Should be called from pytest_collection_modifyitems(). diff --git a/tests/python/ci/test_ci.py b/tests/python/ci/test_ci.py index bfd4b32491..e152d8cb41 100644 --- a/tests/python/ci/test_ci.py +++ b/tests/python/ci/test_ci.py @@ -18,8 +18,6 @@ """Test various CI scripts and GitHub Actions workflows""" import json -import logging -import shutil import subprocess import sys import textwrap @@ -50,277 +48,6 @@ def parameterize_named(**kwargs): ) -# pylint: disable=line-too-long -TEST_DATA_SKIPPED_BOT = { - "found-diff-no-additional": { - "main_xml_file": "unittest/file1.xml", - "main_xml_content": """<?xml version="1.0" encoding="utf-8"?> - <testsuites> - <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102" - tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781"> - <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy" - name="test_sketch_search_policy_cuda_rpc_runner" time="9.679"> - </testcase> - </testsuite> - </testsuites> - """, - "pr_xml_file": "unittest/file2.xml", - "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?> - <testsuites> - <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102" - tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781"> - <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy" - name="test_sketch_search_policy_cuda_rpc_runner" time="9.679"> - <skipped message="This test is skipped" type="pytest.skip"> - Skipped - </skipped> - </testcase> - <testcase classname="ctypes.tests.python.unittest.test_roofline" - name="test_estimate_peak_bandwidth[cuda]" time="4.679"> - <skipped message="This is another skippe test" type="pytest.skip"> - Skipped - </skipped> - </testcase> - </testsuite> - </testsuites> - """, - "additional_tests_to_check": """{ - "unittest": ["dummy_class#dummy_test"], - "unittest_GPU": ["another_dummy_class#another_dummy_test"] - } - """, - "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect";, - "s3_prefix": "tvm-jenkins-artifacts-prod", - "jenkins_prefix": "ci.tlcpack.ai", - "common_main_build": """{"build_number": "4115", "state": "success"}""", - "commit_sha": "sha1234", - "expected_body": "The list below shows tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).", - }, - "found-diff-skipped-additional": { - "main_xml_file": "unittest/file1.xml", - "main_xml_content": """<?xml version="1.0" encoding="utf-8"?> - <testsuites> - <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102" - tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781"> - <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy" - name="test_sketch_search_policy_cuda_rpc_runner" time="9.679"> - </testcase> - </testsuite> - </testsuites> - """, - "pr_xml_file": "unittest/file2.xml", - "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?> - <testsuites> - <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102" - tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781"> - <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy" - name="test_sketch_search_policy_cuda_rpc_runner" time="9.679"> - <skipped message="This test is skipped" type="pytest.skip"> - Skipped - </skipped> - </testcase> - <testcase classname="ctypes.tests.python.unittest.test_roofline" - name="test_estimate_peak_bandwidth[cuda]" time="4.679"> - <skipped message="This is another skippe test" type="pytest.skip"> - Skipped - </skipped> - </testcase> - </testsuite> - </testsuites> - """, - "additional_tests_to_check": """{ - "unittest": ["ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner", "dummy_class#dummy_test"], - "unittest_GPU": ["another_dummy_class#another_dummy_test"] - } - """, - "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect";, - "s3_prefix": "tvm-jenkins-artifacts-prod", - "jenkins_prefix": "ci.tlcpack.ai", - "common_main_build": """{"build_number": "4115", "state": "success"}""", - "commit_sha": "sha1234", - "expected_body": "The list below shows tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/ [...] - }, - "no-diff": { - "main_xml_file": "unittest/file1.xml", - "main_xml_content": """<?xml version="1.0" encoding="utf-8"?> - <testsuites> - <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102" - tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781"> - <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy" - name="test_sketch_search_policy_cuda_rpc_runner" time="9.679"> - <skipped message="This test is skipped" type="pytest.skip"> - Skipped - </skipped> - </testcase> - </testsuite> - </testsuites> - """, - "pr_xml_file": "unittest/file2.xml", - "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?> - <testsuites> - <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102" - tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781"> - <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy" - name="test_sketch_search_policy_cuda_rpc_runner" time="9.679"> - <skipped message="This test is skipped" type="pytest.skip"> - Skipped - </skipped> - </testcase> - </testsuite> - </testsuites> - """, - "additional_tests_to_check": """{ - } - """, - "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect";, - "s3_prefix": "tvm-jenkins-artifacts-prod", - "jenkins_prefix": "ci.tlcpack.ai", - "common_main_build": """{"build_number": "4115", "state": "success"}""", - "commit_sha": "sha1234", - "expected_body": "No diff in skipped tests with main found in this branch for commit sha1234.\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).", - }, - "no-diff-skipped-additional": { - "main_xml_file": "unittest/file1.xml", - "main_xml_content": """<?xml version="1.0" encoding="utf-8"?> - <testsuites> - <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102" - tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781"> - <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy" - name="test_sketch_search_policy_cuda_rpc_runner" time="9.679"> - <skipped message="This test is skipped" type="pytest.skip"> - Skipped - </skipped> - </testcase> - </testsuite> - </testsuites> - """, - "pr_xml_file": "unittest/file2.xml", - "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?> - <testsuites> - <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102" - tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781"> - <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy" - name="test_sketch_search_policy_cuda_rpc_runner" time="9.679"> - <skipped message="This test is skipped" type="pytest.skip"> - Skipped - </skipped> - </testcase> - </testsuite> - </testsuites> - """, - "additional_tests_to_check": """{ - "unittest": ["dummy_class#dummy_test", "ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner"], - "unittest_GPU": ["another_dummy_class#another_dummy_test"] - } - """, - "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect";, - "s3_prefix": "tvm-jenkins-artifacts-prod", - "jenkins_prefix": "ci.tlcpack.ai", - "common_main_build": """{"build_number": "4115", "state": "success"}""", - "commit_sha": "sha1234", - "expected_body": "No diff in skipped tests with main found in this branch for commit sha1234.\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/github/required_tests_to_run.json) file:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm [...] - }, - "unable-to-run": { - "main_xml_file": "unittest/file1.xml", - "main_xml_content": """<?xml version="1.0" encoding="utf-8"?> - <testsuites> - </testsuites> - """, - "pr_xml_file": "unittest/file2.xml", - "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?> - <testsuites> - </testsuites> - """, - "additional_tests_to_check": """{ - "unittest": ["ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner", "dummy_class#dummy_test"], - "unittest_GPU": ["another_dummy_class#another_dummy_test"] - } - """, - "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect";, - "s3_prefix": "tvm-jenkins-artifacts-prod", - "jenkins_prefix": "ci.tlcpack.ai", - "common_main_build": """{"build_number": "4115", "state": "failed"}""", - "commit_sha": "sha1234", - "expected_body": "Unable to run tests bot because main failed to pass CI at sha1234.", - }, -} -# pylint: enable=line-too-long - - [email protected]_if_wheel_test -@parameterize_named(**TEST_DATA_SKIPPED_BOT) -# pylint: enable=line-too-long -def test_skipped_tests_comment( - caplog, - tmpdir_factory, - main_xml_file, - main_xml_content, - pr_xml_file, - pr_xml_content, - additional_tests_to_check, - target_url, - s3_prefix, - jenkins_prefix, - common_main_build, - commit_sha, - expected_body, -): - """ - Test that a comment with a link to the docs is successfully left on PRs - """ - - def write_xml_file(root_dir, xml_file, xml_content): - shutil.rmtree(root_dir, ignore_errors=True) - file = root_dir / xml_file - file.parent.mkdir(parents=True) - with open(file, "w") as f: - f.write(textwrap.dedent(xml_content)) - - git = TempGit(tmpdir_factory.mktemp("tmp_git_dir")) - pr_test_report_dir = Path(git.cwd) / "pr-reports" - write_xml_file(pr_test_report_dir, pr_xml_file, pr_xml_content) - main_test_report_dir = Path(git.cwd) / "main-reports" - write_xml_file(main_test_report_dir, main_xml_file, main_xml_content) - with open(Path(git.cwd) / "required_tests_to_run.json", "w") as f: - f.write(additional_tests_to_check) - - pr_data = { - "commits": { - "nodes": [ - { - "commit": { - "oid": commit_sha, - "statusCheckRollup": { - "contexts": { - "nodes": [ - { - "context": "tvm-ci/pr-head", - "targetUrl": target_url, - } - ] - } - }, - } - } - ] - } - } - with caplog.at_level(logging.INFO): - comment = scripts.github.github_skipped_tests_comment.get_skipped_tests_comment( - pr=pr_data, - github=None, - s3_prefix=s3_prefix, - jenkins_prefix=jenkins_prefix, - common_commit_sha=commit_sha, - pr_test_report_dir=pr_test_report_dir, - main_test_report_dir=main_test_report_dir, - common_main_build=json.loads(common_main_build), - additional_tests_to_check_file=Path(git.cwd) / "required_tests_to_run.json", - ) - assert_in(expected_body, comment) - assert_in(f"with target {target_url}", caplog.text) - - @tvm.testing.skip_if_wheel_test @parameterize_named( doc_link=dict( @@ -548,8 +275,7 @@ def test_update_branch(tmpdir_factory, statuses, expected_rc, expected_output): Thanks for contributing to TVM! Please refer to the contributing guidelines https://tvm.apache.org/docs/contribute/ for useful information and tips. Please request code reviews from [Reviewers](https://github.com/apache/incubator-tvm/blob/master/CONTRIBUTORS.md#reviewers) by @-ing them in a comment. <!--bot-comment-ccs-start--> - * the cc<!--bot-comment-ccs-end--><!--bot-comment-skipped-tests-start--> - * the skipped tests<!--bot-comment-skipped-tests-end--><!--bot-comment-docs-start--> + * the cc<!--bot-comment-ccs-end--><!--bot-comment-docs-start--> * the docs<!--bot-comment-docs-end--> """ ).strip(), @@ -584,8 +310,7 @@ def test_update_branch(tmpdir_factory, statuses, expected_rc, expected_output): <!--bot-comment-ccs-start--> * the cc<!--bot-comment-ccs-end--><!--bot-comment-something-tests-start--> * something else<!--bot-comment-something-tests-end--><!--bot-comment-docs-start--> - * the docs<!--bot-comment-docs-end--><!--bot-comment-skipped-tests-start--> - * the skipped tests<!--bot-comment-skipped-tests-end--> + * the docs<!--bot-comment-docs-end--> """ ).strip(), ), @@ -638,7 +363,6 @@ def test_pr_comment(tmpdir_factory, pr_author, comments, expected): comments = { "ccs": "the cc", "docs": "the docs", - "skipped-tests": "the skipped tests", } proc = run_script( [ diff --git a/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py b/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py index 1e209cacbc..97c9469ccf 100644 --- a/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py +++ b/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py @@ -149,7 +149,7 @@ def _dummy_result(num_samples: int = 4, max_run_sec: int = 10): @requires_xgboost def test_meta_schedule_xgb_model(): extractor = RandomFeatureExtractor() - model = XGBModel(extractor=extractor, num_warmup_samples=2) + model = XGBModel(extractor=extractor, num_warmup_samples=2, num_tuning_cores=1) update_sample_count = 10 predict_sample_count = 100 model.update( @@ -162,9 +162,10 @@ def test_meta_schedule_xgb_model(): @requires_xgboost def test_meta_schedule_xgb_model_no_feature(): - model = XGBModel(num_warmup_samples=0) + model = XGBModel(num_warmup_samples=0, num_tuning_cores=1) tune_ctx = TuneContext( FullModule, + num_threads=1, target={"kind": "llvm", "num-cores": 16}, space_generator="post-order-apply", search_strategy="evolutionary", @@ -177,7 +178,7 @@ def test_meta_schedule_xgb_model_no_feature(): @requires_xgboost def test_meta_schedule_xgb_model_reload(): extractor = RandomFeatureExtractor() - model = XGBModel(extractor=extractor, num_warmup_samples=10) + model = XGBModel(extractor=extractor, num_warmup_samples=10, num_tuning_cores=1) update_sample_count = 20 predict_sample_count = 30 model.update( @@ -221,7 +222,7 @@ def test_meta_schedule_xgb_model_reload(): @requires_xgboost def test_meta_schedule_xgb_model_reupdate(): extractor = RandomFeatureExtractor() - model = XGBModel(extractor=extractor, num_warmup_samples=2) + model = XGBModel(extractor=extractor, num_warmup_samples=2, num_tuning_cores=1) update_sample_count = 60 predict_sample_count = 100 model.update( @@ -252,7 +253,7 @@ def test_meta_schedule_xgb_model_callback_as_function(): # pylint: enable=import-outside-toplevel extractor = RandomFeatureExtractor() - model = XGBModel(extractor=extractor, num_warmup_samples=10) + model = XGBModel(extractor=extractor, num_warmup_samples=10, num_tuning_cores=1) update_sample_count = 20 predict_sample_count = 30 diff --git a/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py b/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py index 002741c6bf..6451a94ec9 100644 --- a/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py +++ b/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py @@ -103,6 +103,7 @@ def test_meta_schedule_replay_func( context = ms.TuneContext( mod=Matmul, + num_threads=1, space_generator=ms.space_generator.ScheduleFn(sch_fn=_schedule_matmul, postprocs=[]), search_strategy=TestClass(), ) diff --git a/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py b/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py index 61f5583c2a..6dc351761a 100644 --- a/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py +++ b/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py @@ -157,6 +157,7 @@ def test_meta_schedule_task_scheduler_single(): [ ms.TuneContext( MatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -183,6 +184,7 @@ def test_meta_schedule_task_scheduler_multiple(): tasks = [ ms.TuneContext( MatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -191,6 +193,7 @@ def test_meta_schedule_task_scheduler_multiple(): ), ms.TuneContext( MatmulReluModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -199,6 +202,7 @@ def test_meta_schedule_task_scheduler_multiple(): ), ms.TuneContext( BatchMatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_batch_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -255,6 +259,7 @@ def test_meta_schedule_task_scheduler_override_next_task_id_only(): # pylint: d tasks = [ ms.TuneContext( MatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -263,6 +268,7 @@ def test_meta_schedule_task_scheduler_override_next_task_id_only(): # pylint: d ), ms.TuneContext( MatmulReluModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -271,6 +277,7 @@ def test_meta_schedule_task_scheduler_override_next_task_id_only(): # pylint: d ), ms.TuneContext( BatchMatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_batch_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -310,6 +317,7 @@ def test_meta_schedule_task_scheduler_multiple_gradient_based(): tasks = [ ms.TuneContext( MatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -318,6 +326,7 @@ def test_meta_schedule_task_scheduler_multiple_gradient_based(): ), ms.TuneContext( MatmulReluModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -326,6 +335,7 @@ def test_meta_schedule_task_scheduler_multiple_gradient_based(): ), ms.TuneContext( BatchMatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_batch_matmul, search_strategy=ms.search_strategy.ReplayTrace(), @@ -394,6 +404,7 @@ def test_meta_schedule_task_scheduler_gradient_based_with_null_search_strategy() tasks = [ ms.TuneContext( MatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=NullSearchStrategy(rounds_with_empty_candidates=5), @@ -402,6 +413,7 @@ def test_meta_schedule_task_scheduler_gradient_based_with_null_search_strategy() ), ms.TuneContext( BatchMatmulModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_batch_matmul, search_strategy=NullSearchStrategy(rounds_with_empty_candidates=0), @@ -410,6 +422,7 @@ def test_meta_schedule_task_scheduler_gradient_based_with_null_search_strategy() ), ms.TuneContext( MatmulReluModule, + num_threads=1, target=tvm.target.Target("llvm"), space_generator=_schedule_matmul, search_strategy=ms.search_strategy.ReplayTrace(), diff --git a/tests/python/testing/test_tvm_testing_features.py b/tests/python/testing/test_tvm_testing_features.py index 549aedda6a..ba2faf2a45 100644 --- a/tests/python/testing/test_tvm_testing_features.py +++ b/tests/python/testing/test_tvm_testing_features.py @@ -14,9 +14,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# ruff: noqa: F401, RUF012 +# ruff: noqa: RUF012 import os +import subprocess import sys import pytest @@ -189,5 +190,44 @@ class TestPytestCache: assert cached_fixture == param * param +def test_empty_marker_selection_is_an_error(tmp_path): + test_file = tmp_path / "test_no_gpu.py" + test_file.write_text("def test_cpu_only():\n pass\n") + env = os.environ.copy() + env.pop("PYTEST_ADDOPTS", None) + + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "tvm.testing.plugin", + "-m", + "gpu", + str(test_file), + ], + env=env, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == pytest.ExitCode.NO_TESTS_COLLECTED, result.stdout + result.stderr + + test_file.write_text("def test_failure():\n assert False\n") + result = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "-p", "tvm.testing.plugin", str(test_file)], + env=env, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == pytest.ExitCode.TESTS_FAILED, result.stdout + result.stderr + assert not list(tmp_path.glob("*.xml")) + + if __name__ == "__main__": tvm.testing.main() diff --git a/tests/scripts/task_clear_pytest.sh b/tests/scripts/task_clear_pytest.sh deleted file mode 100755 index 0a24f62531..0000000000 --- a/tests/scripts/task_clear_pytest.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euxo pipefail - -# Ensure no stale pytest-results remain from a previous test run. -pushd build -rm -rf pytest-results -popd diff --git a/tests/scripts/task_python_nightly.sh b/tests/scripts/task_python_nightly.sh deleted file mode 100755 index 05070f0a04..0000000000 --- a/tests/scripts/task_python_nightly.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euxo pipefail - -export PYTHONPATH="$(pwd)/python" -export PYTEST_ADDOPTS="-s -vv ${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" -mkdir -p build/pytest-results - -# setup tvm-ffi into python folder -uv pip install -v --target=python ./3rdparty/tvm-ffi/ - -# cleanup pycache -find . -type f -path "*.pyc" | xargs rm -f - -python3 -m pytest -n auto \ - -o junit_suite_name=python-topi-nightly \ - --junit-xml=build/pytest-results/python-topi-nightly.xml \ - --junit-prefix=cython \ - tests/python/topi/nightly - -# Tensorflow device verification and network tests on nightly -export CI_ENV_NIGHTLY -python3 tests/python/relax/test_frontend_tflite.py diff --git a/tests/scripts/task_python_unittest.sh b/tests/scripts/task_python_unittest.sh index 14675bf8a0..c05f8503d3 100755 --- a/tests/scripts/task_python_unittest.sh +++ b/tests/scripts/task_python_unittest.sh @@ -20,7 +20,6 @@ set -euxo pipefail export PYTHONPATH="$(pwd)/python" export PYTEST_ADDOPTS="-s -vv ${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" -mkdir -p build/pytest-results # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f @@ -28,28 +27,20 @@ find . -type f -path "*.pyc" | xargs rm -f # setup tvm-ffi into python folder uv pip install -v --target=python ./3rdparty/tvm-ffi/ -# NOTE: also set by task_python_unittest_gpuonly.sh. -if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then - TVM_UNITTEST_TESTSUITE_NAME=python-unittest -fi -PYTEST_SHARD_SUFFIX="${TVM_SHARD_INDEX:+-shard-${TVM_SHARD_INDEX}}" - # First run the minimal platform test. A GPU-only run can select no tests here. -python3 -m pytest -n auto \ - -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test${PYTEST_SHARD_SUFFIX}" \ - "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test${PYTEST_SHARD_SUFFIX}.xml" \ - --junit-prefix=cython \ - tests/python/all-platform-minimal-test || [ "$?" -eq 5 ] +if [ ! -d tests/python/all-platform-minimal-test ]; then + echo "Missing pytest target: tests/python/all-platform-minimal-test" >&2 + exit 1 +fi +python3 -m pytest -n auto tests/python/all-platform-minimal-test || [ "$?" -eq 5 ] # Then run all unit tests. TEST_FILES=( - "ffi" "arith" "ci" "codegen" "driver" "ir" - "meta_schedule" "runtime" "target" "te" @@ -58,6 +49,7 @@ TEST_FILES=( "s_tir/schedule" "s_tir/dlight" "s_tir/analysis" + "s_tir/meta_schedule" "s_tir/transform" "tirx-analysis" "tirx-base" @@ -69,11 +61,13 @@ TEST_FILES=( PYTEST_TARGETS=() for TEST_FILE in "${TEST_FILES[@]}"; do - PYTEST_TARGETS+=("tests/python/${TEST_FILE}") + TEST_PATH="tests/python/${TEST_FILE}" + if [ ! -d "${TEST_PATH}" ]; then + echo "Missing pytest target: ${TEST_PATH}" >&2 + exit 1 + fi + PYTEST_TARGETS+=("${TEST_PATH}") done -python3 -m pytest -n auto --dist=loadgroup \ - -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}${PYTEST_SHARD_SUFFIX}" \ - "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}${PYTEST_SHARD_SUFFIX}.xml" \ - --junit-prefix=cython \ - "${PYTEST_TARGETS[@]}" +# Do not mask pytest's exit 5: an unexpectedly empty broad suite must fail CI. +python3 -m pytest -n auto --dist=loadgroup "${PYTEST_TARGETS[@]}" diff --git a/tests/scripts/task_python_unittest_gpuonly.sh b/tests/scripts/task_python_unittest_gpuonly.sh index 9753a1a8a4..8194b23ac4 100755 --- a/tests/scripts/task_python_unittest_gpuonly.sh +++ b/tests/scripts/task_python_unittest_gpuonly.sh @@ -24,22 +24,18 @@ export PYTEST_ADDOPTS="-m gpu ${PYTEST_ADDOPTS:-}" # Test most of the enabled runtimes here. # TODO: disabled opencl tests due to segmentation fault. export TVM_TEST_TARGETS='cuda;metal;rocm;nvptx' -export TVM_UNITTEST_TESTSUITE_NAME=python-unittest-gpu ./tests/scripts/task_python_unittest.sh # Kept separate to avoid increasing time needed to run CI, testing # only minimal functionality of Vulkan runtime. export TVM_TEST_TARGETS='{"kind":"vulkan","from_device":0}' -export TVM_UNITTEST_TESTSUITE_NAME=python-codegen-vulkan export PYTHONPATH="$(pwd)/python" export PYTEST_ADDOPTS="-s -vv ${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}" -mkdir -p build/pytest-results -PYTEST_SHARD_SUFFIX="${TVM_SHARD_INDEX:+-shard-${TVM_SHARD_INDEX}}" -python3 -m pytest -n auto \ - -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}${PYTEST_SHARD_SUFFIX}" \ - "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}${PYTEST_SHARD_SUFFIX}.xml" \ - --junit-prefix=cython \ - tests/python/codegen/test_target_codegen_vulkan.py +if [ ! -f tests/python/codegen/test_target_codegen_vulkan.py ]; then + echo "Missing pytest target: tests/python/codegen/test_target_codegen_vulkan.py" >&2 + exit 1 +fi +python3 -m pytest -n auto tests/python/codegen/test_target_codegen_vulkan.py
