This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch tvm-simplify-jenkins-pytest-sharding
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to
refs/heads/tvm-simplify-jenkins-pytest-sharding by this push:
new 9f4feee101 [CI] Remove pytest JUnit reporting
9f4feee101 is described below
commit 9f4feee10113767bcec5676ba6b9e1c0ebdae4ac
Author: Tianqi Chen <[email protected]>
AuthorDate: Sun Jul 5 01:36:07 2026 +0000
[CI] Remove pytest JUnit reporting
---
ci/jenkins/generated/arm_jenkinsfile.groovy | 9 +-
ci/jenkins/generated/cpu_jenkinsfile.groovy | 21 +-
ci/jenkins/generated/docker_jenkinsfile.groovy | 9 +-
ci/jenkins/generated/gpu_jenkinsfile.groovy | 33 +--
ci/jenkins/generated/wasm_jenkinsfile.groovy | 10 +-
ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 | 1 -
ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 | 2 -
ci/jenkins/templates/utils/Build.groovy.j2 | 7 -
ci/jenkins/templates/utils/macros.j2 | 15 --
ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 | 1 -
ci/scripts/github/__init__.py | 2 +-
ci/scripts/github/github_pr_comment.py | 4 -
ci/scripts/github/github_skipped_tests_comment.py | 263 -------------------
python/tvm/testing/plugin.py | 9 -
tests/python/ci/test_ci.py | 280 +--------------------
.../meta_schedule/test_meta_schedule_cost_model.py | 11 +-
.../test_meta_schedule_search_strategy.py | 1 +
.../test_meta_schedule_task_scheduler.py | 13 +
tests/python/testing/test_tvm_testing_features.py | 42 +++-
tests/scripts/task_clear_pytest.sh | 24 --
tests/scripts/task_python_nightly.sh | 39 ---
tests/scripts/task_python_unittest.sh | 33 ++-
tests/scripts/task_python_unittest_gpuonly.sh | 13 +-
23 files changed, 88 insertions(+), 754 deletions(-)
diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy
b/ci/jenkins/generated/arm_jenkinsfile.groovy
index 573885545a..3442caded7 100644
--- a/ci/jenkins/generated/arm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/arm_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2026-07-04T23:53:56.833713
+// Generated at 2026-07-05T01:05:57.719405
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -404,13 +404,6 @@ def prepare(node_type) {
}
}
}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
def python_unittest(image) {
sh (
script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy
b/ci/jenkins/generated/cpu_jenkinsfile.groovy
index dc944096ab..0d3e408235 100644
--- a/ci/jenkins/generated/cpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2026-07-04T23:53:56.855205
+// Generated at 2026-07-05T01:05:57.741181
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -404,13 +404,6 @@ def prepare(node_type) {
}
}
}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
def python_unittest(image) {
sh (
script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
@@ -528,22 +521,10 @@ def run_unittest_CPU(node_type) {
label: 'Download artifacts from S3',
)
- ci_setup(ci_cpu)
cpp_unittest(ci_cpu)
python_unittest(ci_cpu)
})
}
- // only run upload if things are successful
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU --items
build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
}
}
echo 'End running on node_type ' + node_type
diff --git a/ci/jenkins/generated/docker_jenkinsfile.groovy
b/ci/jenkins/generated/docker_jenkinsfile.groovy
index 95f2fd8dd6..8ffe78942a 100644
--- a/ci/jenkins/generated/docker_jenkinsfile.groovy
+++ b/ci/jenkins/generated/docker_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2026-07-04T23:53:56.875051
+// Generated at 2026-07-05T01:05:57.759905
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -404,13 +404,6 @@ def prepare(node_type) {
}
}
}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
def python_unittest(image) {
sh (
script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy
b/ci/jenkins/generated/gpu_jenkinsfile.groovy
index d31160dd52..792b7eadc4 100644
--- a/ci/jenkins/generated/gpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2026-07-04T23:53:56.896868
+// Generated at 2026-07-05T01:05:57.780838
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -404,13 +404,6 @@ def prepare(node_type) {
}
}
}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
def python_unittest(image) {
sh (
script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
@@ -533,7 +526,6 @@ def run_unittest_GPU(node_type) {
label: 'Download artifacts from S3',
)
- ci_setup(ci_gpu)
sh (
script: "${docker_run} ${ci_gpu}
./tests/scripts/task_java_unittest.sh",
label: 'Run Java unit tests',
@@ -548,17 +540,6 @@ def run_unittest_GPU(node_type) {
)
})
}
- // only run upload if things are successful
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items
build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
}
}
echo 'End running on node_type ' + node_type
@@ -587,7 +568,6 @@ def run_docs_GPU(node_type) {
label: 'Download artifacts from S3',
)
- ci_setup(ci_gpu)
sh (
script: "${docker_run} ${ci_gpu}
./tests/scripts/task_python_docs.sh",
label: 'Build docs',
@@ -603,17 +583,6 @@ def run_docs_GPU(node_type) {
)
})
}
- // only run upload if things are successful
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/pytest-results/docs_GPU --items
build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
}
}
echo 'End running on node_type ' + node_type
diff --git a/ci/jenkins/generated/wasm_jenkinsfile.groovy
b/ci/jenkins/generated/wasm_jenkinsfile.groovy
index b6946a40fe..7421fa86ca 100644
--- a/ci/jenkins/generated/wasm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2026-07-04T23:53:56.918767
+// Generated at 2026-07-05T01:05:57.802104
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -404,13 +404,6 @@ def prepare(node_type) {
}
}
}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
def python_unittest(image) {
sh (
script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
@@ -471,7 +464,6 @@ def run_build(node_type) {
cmake_build(ci_wasm, 'build')
make_cpp_tests(ci_wasm, 'build')
cpp_unittest(ci_wasm)
- ci_setup(ci_wasm)
sh (
script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh",
label: 'Run WASM lint and tests',
diff --git a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
index 75b43c52a8..62b7c3dfea 100644
--- a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
@@ -45,7 +45,6 @@
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='cpu') }}
- ci_setup(ci_cpu)
cpp_unittest(ci_cpu)
python_unittest(ci_cpu)
{% endcall %}
diff --git a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
index 33e5a8aed5..3a13dfe7f3 100644
--- a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
@@ -46,7 +46,6 @@
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='gpu') }}
- ci_setup(ci_gpu)
sh (
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh",
label: 'Run Java unit tests',
@@ -71,7 +70,6 @@
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='gpu') }}
- ci_setup(ci_gpu)
sh (
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh",
label: 'Build docs',
diff --git a/ci/jenkins/templates/utils/Build.groovy.j2
b/ci/jenkins/templates/utils/Build.groovy.j2
index c20a1c5c43..42bd4a991f 100644
--- a/ci/jenkins/templates/utils/Build.groovy.j2
+++ b/ci/jenkins/templates/utils/Build.groovy.j2
@@ -1,10 +1,3 @@
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
def python_unittest(image) {
sh (
script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
diff --git a/ci/jenkins/templates/utils/macros.j2
b/ci/jenkins/templates/utils/macros.j2
index bd0e0d4ee4..959d91725f 100644
--- a/ci/jenkins/templates/utils/macros.j2
+++ b/ci/jenkins/templates/utils/macros.j2
@@ -19,16 +19,8 @@
"workspace/exec_${env.EXECUTOR_NUMBER}/{{ folder }}"
{%- endmacro -%}
-{% macro junit_to_s3(test_dir_name) %}
-sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket
${s3_bucket} --prefix ${s3_prefix}/pytest-results/{{ test_dir_name }} --items
build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-{% endmacro %}
-
{% macro test_step(name, ws, docker_image, platform, test_method_names,
condition="!skip_ci && is_docs_only_build != 1") %}
{% set method_name = "run_" + name.replace(":", "").replace(" ",
"-").replace("-", "_") %}
-{% set test_dir_name = name.replace(":", "").replace(" ", "-").replace("-",
"_")|string %}
def {{ method_name }}(node_type) {
echo 'Begin running on node_type ' + node_type
if ({{ condition }}) {
@@ -45,13 +37,6 @@ def {{ method_name }}(node_type) {
{{ caller() | trim | indent(width=12) }}
})
}
- // only run upload if things are successful
- try {
- {{ junit_to_s3(test_dir_name) }}
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
}
}
echo 'End running on node_type ' + node_type
diff --git a/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2
b/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2
index 0d119f043e..8e7d909bd1 100644
--- a/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2
@@ -32,7 +32,6 @@
cmake_build(ci_wasm, 'build')
make_cpp_tests(ci_wasm, 'build')
cpp_unittest(ci_wasm)
- ci_setup(ci_wasm)
sh (
script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh",
label: 'Run WASM lint and tests',
diff --git a/ci/scripts/github/__init__.py b/ci/scripts/github/__init__.py
index a6226fec29..0e55424eee 100644
--- a/ci/scripts/github/__init__.py
+++ b/ci/scripts/github/__init__.py
@@ -17,4 +17,4 @@
# under the License.
"""Package to enable testing of GitHub scripts"""
-from . import github_skipped_tests_comment, github_pr_comment,
github_tag_teams, github_docs_comment
+from . import github_docs_comment, github_pr_comment, github_tag_teams
diff --git a/ci/scripts/github/github_pr_comment.py
b/ci/scripts/github/github_pr_comment.py
index 2d523b21b3..81347659d9 100755
--- a/ci/scripts/github/github_pr_comment.py
+++ b/ci/scripts/github/github_pr_comment.py
@@ -30,7 +30,6 @@ from cmd_utils import init_log
from git_utils import DRY_RUN, GitHubRepo, git, parse_remote
from github_commenter import BotCommentBuilder
from github_docs_comment import get_doc_url
-from github_skipped_tests_comment import get_skipped_tests_comment
from github_tag_teams import get_tags
PR_QUERY = """
@@ -132,17 +131,14 @@ if __name__ == "__main__":
if args.test_comments is not None:
test_comments = json.loads(args.test_comments)
- skipped_tests = test_comments["skipped-tests"]
ccs = test_comments["ccs"]
docs_info = test_comments["docs"]
else:
- skipped_tests = get_skipped_tests_comment(pr_data, github=github)
ccs = get_tags(pr_data, github, team_issue=10317)
docs_info = get_doc_url(pr_data)
items = {
"ccs": ccs,
- "skipped-tests": skipped_tests,
"docs": docs_info,
}
commenter.post_items(items=items.items())
diff --git a/ci/scripts/github/github_skipped_tests_comment.py
b/ci/scripts/github/github_skipped_tests_comment.py
deleted file mode 100755
index 10789f7561..0000000000
--- a/ci/scripts/github/github_skipped_tests_comment.py
+++ /dev/null
@@ -1,263 +0,0 @@
-#!/usr/bin/env python3
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# ruff: noqa: E501
-import json
-import logging
-import os
-import subprocess
-from pathlib import Path
-from typing import Any
-from xml.etree import ElementTree
-
-
-def run_subprocess(command):
- logging.info(f"Running command {command}")
- proc = subprocess.run(command, shell=True, stdout=subprocess.PIPE,
encoding="utf-8")
- if proc.returncode != 0:
- raise RuntimeError(f"Command failed
{command}:\nstdout:\n{proc.stdout}")
- return proc
-
-
-def retrieve_test_report(s3_url, target_dir):
- command = f"aws --region us-west-2 s3 cp {s3_url} {target_dir} --recursive
--no-sign-request"
- run_subprocess(command)
-
-
-def get_common_commit_sha():
- command = "git merge-base origin/main HEAD"
- proc = run_subprocess(command)
- return proc.stdout.strip()
-
-
-def get_main_jenkins_build_number(github, common_commit):
- json = github.get(f"commits/{common_commit}/status")
- for status in reversed(json["statuses"]):
- if status["context"] != "tvm-ci/branch":
- continue
- state = status["state"]
- target_url = str(status["target_url"])
- build_number = (
- target_url[target_url.find("job/main") : len(target_url)]
- .strip("job/main/")
- .strip("/display/redirect")
- )
- assert build_number.isdigit()
- return {"build_number": build_number, "state": state}
- raise RuntimeError(f"Failed to find main build number for commit
{common_commit}")
-
-
-def retrieve_test_reports(
- common_main_build, pr_number, build_number, s3_prefix, pr_test_report_dir,
main_test_report_dir
-):
- cur_build_s3_link =
f"s3://{s3_prefix}/tvm/PR-{pr_number!s}/{build_number!s}/pytest-results"
- retrieve_test_report(cur_build_s3_link, pr_test_report_dir)
-
- common_build_s3_link =
f"s3://{s3_prefix}/tvm/main/{common_main_build}/pytest-results"
- retrieve_test_report(common_build_s3_link, main_test_report_dir)
-
-
-def get_pr_and_build_numbers(target_url):
- target_url = target_url[target_url.find("PR-") : len(target_url)]
- split = target_url.split("/")
- pr_number = split[0].strip("PR-")
- build_number = split[1]
- return {"pr_number": pr_number, "build_number": build_number}
-
-
-def build_test_set(directory):
- directory = Path(directory)
- subdir_to_skipped = {}
- subdirs = [
- item for item in os.listdir(directory) if
os.path.isdir(os.path.join(directory, item))
- ]
- for subdir in subdirs:
- subdir_to_skipped[subdir] = set()
- for root, _, files in os.walk(directory / subdir):
- for file in files:
- test_report = ElementTree.parse(Path(root) / file)
- for testcase in test_report.iter("testcase"):
- skipped = testcase.find("skipped")
- if skipped is not None:
- key = testcase.attrib["classname"] + "#" +
testcase.attrib["name"]
- subdir_to_skipped[subdir].add(key)
- return subdir_to_skipped
-
-
-def to_node_name(dir_name: str):
- return dir_name.replace("_", ": ", 1)
-
-
-def build_diff_comment_with_main(
- common_commit_sha,
- skipped_list,
- commit_sha,
-):
- if len(skipped_list) == 0:
- return f"No diff in skipped tests with main found in this branch for
commit {commit_sha}.\n"
-
- text = (
- f"The list below shows tests that ran in main {common_commit_sha} but
were "
- f"skipped in the CI build of {commit_sha}:\n"
- f"```\n"
- )
- for skip in skipped_list:
- text += skip + "\n"
- text += "```\n"
- return text
-
-
-def build_comment(
- common_commit_sha,
- common_main_build,
- skipped_list,
- additional_skipped_list,
- pr_number,
- build_number,
- commit_sha,
- jenkins_prefix,
-):
- if common_main_build["state"] != "success":
- return f"Unable to run tests bot because main failed to pass CI at
{common_commit_sha}."
-
- text = build_diff_comment_with_main(common_commit_sha, skipped_list,
commit_sha)
-
- if len(additional_skipped_list) != 0:
- text += "\n"
- text += (
- "Additional tests that were skipped in the CI build and present in
the [`required_tests_to_run`]"
-
"(https://github.com/apache/tvm/blob/main/ci/scripts/github/required_tests_to_run.json)
file:"
- "\n```\n"
- )
- for skip in additional_skipped_list:
- text += skip + "\n"
- text += "```\n"
-
- text += (
- f"A detailed report of ran tests is
[here](https://{jenkins_prefix}/job/tvm/job/PR-{pr_number!s}";
- f"/{build_number!s}/testReport/)."
- )
- return text
-
-
-def find_target_url(pr_head: dict[str, Any]):
- for status in pr_head["statusCheckRollup"]["contexts"]["nodes"]:
- if status.get("context", "") == "tvm-ci/pr-head":
- return status["targetUrl"]
-
- raise RuntimeError(f"Unable to find tvm-ci/pr-head status in {pr_head}")
-
-
-def get_skipped_tests_comment(
- pr: dict[str, Any],
- github,
- s3_prefix: str = "tvm-jenkins-artifacts-prod",
- jenkins_prefix: str = "ci.tlcpack.ai",
- pr_test_report_dir: str = "pr-reports",
- main_test_report_dir: str = "main-reports",
- common_commit_sha: str | None = None,
- common_main_build: dict[str, Any] | None = None,
- additional_tests_to_check_file: str = "required_tests_to_run.json",
-) -> str:
- pr_head = pr["commits"]["nodes"][0]["commit"]
- target_url = find_target_url(pr_head)
- pr_and_build = get_pr_and_build_numbers(target_url)
- logging.info(f"Getting comment for {pr_head} with target {target_url}")
-
- commit_sha = pr_head["oid"]
-
- is_dry_run = common_commit_sha is not None
-
- if not is_dry_run:
- logging.info("Fetching common commit sha and build info")
- common_commit_sha = get_common_commit_sha()
- common_main_build = get_main_jenkins_build_number(github,
common_commit_sha)
-
- retrieve_test_reports(
- common_main_build=common_main_build["build_number"],
- pr_number=pr_and_build["pr_number"],
- build_number=pr_and_build["build_number"],
- s3_prefix=s3_prefix,
- main_test_report_dir=main_test_report_dir,
- pr_test_report_dir=pr_test_report_dir,
- )
- else:
- logging.info("Dry run, expecting PR and main reports on disk")
-
- main_tests = build_test_set(main_test_report_dir)
- build_tests = build_test_set(pr_test_report_dir)
-
- skipped_list = []
- for subdir, skipped_set in build_tests.items():
- skipped_main = main_tests[subdir]
- if skipped_main is None:
- logging.warning(f"Could not find directory {subdir} in main.")
- continue
-
- diff_set = skipped_set - skipped_main
- if len(diff_set) != 0:
- for test in diff_set:
- skipped_list.append(f"{to_node_name(subdir)} -> {test}")
-
- # Sort the list to maintain an order in the output. Helps when validating
the output in tests.
- skipped_list.sort()
-
- if len(skipped_list) == 0:
- logging.info("No skipped tests found.")
-
- if not is_dry_run:
- current_file = Path(__file__).resolve()
- additional_tests_to_check_file = Path(current_file).parent /
"required_tests_to_run.json"
-
- logging.info(
- f"Checking additional tests in file {additional_tests_to_check_file}
are not skipped."
- )
- try:
- with open(additional_tests_to_check_file) as f:
- additional_tests_to_check = json.load(f)
- except OSError:
- logging.info(
- f"Failed to read additional tests from file:
{additional_tests_to_check_file}."
- )
- additional_tests_to_check = {}
-
- # Assert that tests present in "required_tests_to_run.json" are not
skipped.
- additional_skipped_tests = []
- for subdir, test_set in additional_tests_to_check.items():
- if subdir not in build_tests.keys():
- logging.warning(f"Could not find directory {subdir} in the build
test set.")
- continue
-
- for test in test_set:
- if test in build_tests[subdir]:
- additional_skipped_tests.append(f"{to_node_name(subdir)} ->
{test}")
-
- if len(additional_skipped_tests) == 0:
- logging.info("No skipped tests found in the additional list.")
-
- body = build_comment(
- common_commit_sha,
- common_main_build,
- skipped_list,
- additional_skipped_tests,
- pr_and_build["pr_number"],
- pr_and_build["build_number"],
- commit_sha,
- jenkins_prefix,
- )
-
- return body
diff --git a/python/tvm/testing/plugin.py b/python/tvm/testing/plugin.py
index dfb858f1b8..2869664842 100644
--- a/python/tvm/testing/plugin.py
+++ b/python/tvm/testing/plugin.py
@@ -33,7 +33,6 @@ directory as the test scripts.
"""
import _pytest
-import pytest
def pytest_collection_modifyitems(config, items):
@@ -44,14 +43,6 @@ def pytest_collection_modifyitems(config, items):
_sort_tests(items)
-def pytest_sessionfinish(session, exitstatus):
- # Don't exit with an error if we select a subset of tests that doesn't
- # include anything
- if session.config.option.markexpr != "":
- if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED:
- session.exitstatus = pytest.ExitCode.OK
-
-
def _count_num_fixture_uses(items):
# Helper function, counts the number of tests that use each cached
# fixture. Should be called from pytest_collection_modifyitems().
diff --git a/tests/python/ci/test_ci.py b/tests/python/ci/test_ci.py
index bfd4b32491..e152d8cb41 100644
--- a/tests/python/ci/test_ci.py
+++ b/tests/python/ci/test_ci.py
@@ -18,8 +18,6 @@
"""Test various CI scripts and GitHub Actions workflows"""
import json
-import logging
-import shutil
import subprocess
import sys
import textwrap
@@ -50,277 +48,6 @@ def parameterize_named(**kwargs):
)
-# pylint: disable=line-too-long
-TEST_DATA_SKIPPED_BOT = {
- "found-diff-no-additional": {
- "main_xml_file": "unittest/file1.xml",
- "main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8"
name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312"
timestamp="2022-08-10T22:39:36.673781">
- <testcase
classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
-
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "pr_xml_file": "unittest/file2.xml",
- "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8"
name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312"
timestamp="2022-08-10T22:39:36.673781">
- <testcase
classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
-
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- <skipped message="This test is skipped"
type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- <testcase
classname="ctypes.tests.python.unittest.test_roofline"
- name="test_estimate_peak_bandwidth[cuda]"
time="4.679">
- <skipped message="This is another skippe test"
type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "additional_tests_to_check": """{
- "unittest": ["dummy_class#dummy_test"],
- "unittest_GPU": ["another_dummy_class#another_dummy_test"]
- }
- """,
- "target_url":
"https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect";,
- "s3_prefix": "tvm-jenkins-artifacts-prod",
- "jenkins_prefix": "ci.tlcpack.ai",
- "common_main_build": """{"build_number": "4115", "state":
"success"}""",
- "commit_sha": "sha1234",
- "expected_body": "The list below shows tests that ran in main sha1234
but were skipped in the CI build of sha1234:\n```\nunittest ->
ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest
->
ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\nA
detailed report of ran tests is
[here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
- },
- "found-diff-skipped-additional": {
- "main_xml_file": "unittest/file1.xml",
- "main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8"
name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312"
timestamp="2022-08-10T22:39:36.673781">
- <testcase
classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
-
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "pr_xml_file": "unittest/file2.xml",
- "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8"
name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312"
timestamp="2022-08-10T22:39:36.673781">
- <testcase
classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
-
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- <skipped message="This test is skipped"
type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- <testcase
classname="ctypes.tests.python.unittest.test_roofline"
- name="test_estimate_peak_bandwidth[cuda]"
time="4.679">
- <skipped message="This is another skippe test"
type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "additional_tests_to_check": """{
- "unittest":
["ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner",
"dummy_class#dummy_test"],
- "unittest_GPU": ["another_dummy_class#another_dummy_test"]
- }
- """,
- "target_url":
"https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect";,
- "s3_prefix": "tvm-jenkins-artifacts-prod",
- "jenkins_prefix": "ci.tlcpack.ai",
- "common_main_build": """{"build_number": "4115", "state":
"success"}""",
- "commit_sha": "sha1234",
- "expected_body": "The list below shows tests that ran in main sha1234
but were skipped in the CI build of sha1234:\n```\nunittest ->
ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest
->
ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\n\nAdditional
tests that were skipped in the CI build and present in the
[`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/
[...]
- },
- "no-diff": {
- "main_xml_file": "unittest/file1.xml",
- "main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8"
name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312"
timestamp="2022-08-10T22:39:36.673781">
- <testcase
classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
-
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- <skipped message="This test is skipped"
type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "pr_xml_file": "unittest/file2.xml",
- "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8"
name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312"
timestamp="2022-08-10T22:39:36.673781">
- <testcase
classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
-
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- <skipped message="This test is skipped"
type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "additional_tests_to_check": """{
- }
- """,
- "target_url":
"https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect";,
- "s3_prefix": "tvm-jenkins-artifacts-prod",
- "jenkins_prefix": "ci.tlcpack.ai",
- "common_main_build": """{"build_number": "4115", "state":
"success"}""",
- "commit_sha": "sha1234",
- "expected_body": "No diff in skipped tests with main found in this
branch for commit sha1234.\nA detailed report of ran tests is
[here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
- },
- "no-diff-skipped-additional": {
- "main_xml_file": "unittest/file1.xml",
- "main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8"
name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312"
timestamp="2022-08-10T22:39:36.673781">
- <testcase
classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
-
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- <skipped message="This test is skipped"
type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "pr_xml_file": "unittest/file2.xml",
- "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8"
name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312"
timestamp="2022-08-10T22:39:36.673781">
- <testcase
classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
-
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- <skipped message="This test is skipped"
type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "additional_tests_to_check": """{
- "unittest": ["dummy_class#dummy_test",
"ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner"],
- "unittest_GPU": ["another_dummy_class#another_dummy_test"]
- }
- """,
- "target_url":
"https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect";,
- "s3_prefix": "tvm-jenkins-artifacts-prod",
- "jenkins_prefix": "ci.tlcpack.ai",
- "common_main_build": """{"build_number": "4115", "state":
"success"}""",
- "commit_sha": "sha1234",
- "expected_body": "No diff in skipped tests with main found in this
branch for commit sha1234.\n\nAdditional tests that were skipped in the CI
build and present in the
[`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/github/required_tests_to_run.json)
file:\n```\nunittest ->
ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\n```\nA
detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm [...]
- },
- "unable-to-run": {
- "main_xml_file": "unittest/file1.xml",
- "main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- </testsuites>
- """,
- "pr_xml_file": "unittest/file2.xml",
- "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- </testsuites>
- """,
- "additional_tests_to_check": """{
- "unittest":
["ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner",
"dummy_class#dummy_test"],
- "unittest_GPU": ["another_dummy_class#another_dummy_test"]
- }
- """,
- "target_url":
"https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect";,
- "s3_prefix": "tvm-jenkins-artifacts-prod",
- "jenkins_prefix": "ci.tlcpack.ai",
- "common_main_build": """{"build_number": "4115", "state": "failed"}""",
- "commit_sha": "sha1234",
- "expected_body": "Unable to run tests bot because main failed to pass
CI at sha1234.",
- },
-}
-# pylint: enable=line-too-long
-
-
[email protected]_if_wheel_test
-@parameterize_named(**TEST_DATA_SKIPPED_BOT)
-# pylint: enable=line-too-long
-def test_skipped_tests_comment(
- caplog,
- tmpdir_factory,
- main_xml_file,
- main_xml_content,
- pr_xml_file,
- pr_xml_content,
- additional_tests_to_check,
- target_url,
- s3_prefix,
- jenkins_prefix,
- common_main_build,
- commit_sha,
- expected_body,
-):
- """
- Test that a comment with a link to the docs is successfully left on PRs
- """
-
- def write_xml_file(root_dir, xml_file, xml_content):
- shutil.rmtree(root_dir, ignore_errors=True)
- file = root_dir / xml_file
- file.parent.mkdir(parents=True)
- with open(file, "w") as f:
- f.write(textwrap.dedent(xml_content))
-
- git = TempGit(tmpdir_factory.mktemp("tmp_git_dir"))
- pr_test_report_dir = Path(git.cwd) / "pr-reports"
- write_xml_file(pr_test_report_dir, pr_xml_file, pr_xml_content)
- main_test_report_dir = Path(git.cwd) / "main-reports"
- write_xml_file(main_test_report_dir, main_xml_file, main_xml_content)
- with open(Path(git.cwd) / "required_tests_to_run.json", "w") as f:
- f.write(additional_tests_to_check)
-
- pr_data = {
- "commits": {
- "nodes": [
- {
- "commit": {
- "oid": commit_sha,
- "statusCheckRollup": {
- "contexts": {
- "nodes": [
- {
- "context": "tvm-ci/pr-head",
- "targetUrl": target_url,
- }
- ]
- }
- },
- }
- }
- ]
- }
- }
- with caplog.at_level(logging.INFO):
- comment =
scripts.github.github_skipped_tests_comment.get_skipped_tests_comment(
- pr=pr_data,
- github=None,
- s3_prefix=s3_prefix,
- jenkins_prefix=jenkins_prefix,
- common_commit_sha=commit_sha,
- pr_test_report_dir=pr_test_report_dir,
- main_test_report_dir=main_test_report_dir,
- common_main_build=json.loads(common_main_build),
- additional_tests_to_check_file=Path(git.cwd) /
"required_tests_to_run.json",
- )
- assert_in(expected_body, comment)
- assert_in(f"with target {target_url}", caplog.text)
-
-
@tvm.testing.skip_if_wheel_test
@parameterize_named(
doc_link=dict(
@@ -548,8 +275,7 @@ def test_update_branch(tmpdir_factory, statuses,
expected_rc, expected_output):
Thanks for contributing to TVM! Please refer to the contributing
guidelines https://tvm.apache.org/docs/contribute/ for useful information and
tips. Please request code reviews from
[Reviewers](https://github.com/apache/incubator-tvm/blob/master/CONTRIBUTORS.md#reviewers)
by @-ing them in a comment.
<!--bot-comment-ccs-start-->
- * the
cc<!--bot-comment-ccs-end--><!--bot-comment-skipped-tests-start-->
- * the skipped
tests<!--bot-comment-skipped-tests-end--><!--bot-comment-docs-start-->
+ * the cc<!--bot-comment-ccs-end--><!--bot-comment-docs-start-->
* the docs<!--bot-comment-docs-end-->
"""
).strip(),
@@ -584,8 +310,7 @@ def test_update_branch(tmpdir_factory, statuses,
expected_rc, expected_output):
<!--bot-comment-ccs-start-->
* the
cc<!--bot-comment-ccs-end--><!--bot-comment-something-tests-start-->
* something
else<!--bot-comment-something-tests-end--><!--bot-comment-docs-start-->
- * the
docs<!--bot-comment-docs-end--><!--bot-comment-skipped-tests-start-->
- * the skipped tests<!--bot-comment-skipped-tests-end-->
+ * the docs<!--bot-comment-docs-end-->
"""
).strip(),
),
@@ -638,7 +363,6 @@ def test_pr_comment(tmpdir_factory, pr_author, comments,
expected):
comments = {
"ccs": "the cc",
"docs": "the docs",
- "skipped-tests": "the skipped tests",
}
proc = run_script(
[
diff --git a/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py
b/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py
index 1e209cacbc..97c9469ccf 100644
--- a/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py
+++ b/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py
@@ -149,7 +149,7 @@ def _dummy_result(num_samples: int = 4, max_run_sec: int =
10):
@requires_xgboost
def test_meta_schedule_xgb_model():
extractor = RandomFeatureExtractor()
- model = XGBModel(extractor=extractor, num_warmup_samples=2)
+ model = XGBModel(extractor=extractor, num_warmup_samples=2,
num_tuning_cores=1)
update_sample_count = 10
predict_sample_count = 100
model.update(
@@ -162,9 +162,10 @@ def test_meta_schedule_xgb_model():
@requires_xgboost
def test_meta_schedule_xgb_model_no_feature():
- model = XGBModel(num_warmup_samples=0)
+ model = XGBModel(num_warmup_samples=0, num_tuning_cores=1)
tune_ctx = TuneContext(
FullModule,
+ num_threads=1,
target={"kind": "llvm", "num-cores": 16},
space_generator="post-order-apply",
search_strategy="evolutionary",
@@ -177,7 +178,7 @@ def test_meta_schedule_xgb_model_no_feature():
@requires_xgboost
def test_meta_schedule_xgb_model_reload():
extractor = RandomFeatureExtractor()
- model = XGBModel(extractor=extractor, num_warmup_samples=10)
+ model = XGBModel(extractor=extractor, num_warmup_samples=10,
num_tuning_cores=1)
update_sample_count = 20
predict_sample_count = 30
model.update(
@@ -221,7 +222,7 @@ def test_meta_schedule_xgb_model_reload():
@requires_xgboost
def test_meta_schedule_xgb_model_reupdate():
extractor = RandomFeatureExtractor()
- model = XGBModel(extractor=extractor, num_warmup_samples=2)
+ model = XGBModel(extractor=extractor, num_warmup_samples=2,
num_tuning_cores=1)
update_sample_count = 60
predict_sample_count = 100
model.update(
@@ -252,7 +253,7 @@ def test_meta_schedule_xgb_model_callback_as_function():
# pylint: enable=import-outside-toplevel
extractor = RandomFeatureExtractor()
- model = XGBModel(extractor=extractor, num_warmup_samples=10)
+ model = XGBModel(extractor=extractor, num_warmup_samples=10,
num_tuning_cores=1)
update_sample_count = 20
predict_sample_count = 30
diff --git
a/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py
b/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py
index 002741c6bf..6451a94ec9 100644
--- a/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py
+++ b/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py
@@ -103,6 +103,7 @@ def test_meta_schedule_replay_func(
context = ms.TuneContext(
mod=Matmul,
+ num_threads=1,
space_generator=ms.space_generator.ScheduleFn(sch_fn=_schedule_matmul,
postprocs=[]),
search_strategy=TestClass(),
)
diff --git
a/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py
b/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py
index 61f5583c2a..6dc351761a 100644
--- a/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py
+++ b/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py
@@ -157,6 +157,7 @@ def test_meta_schedule_task_scheduler_single():
[
ms.TuneContext(
MatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -183,6 +184,7 @@ def test_meta_schedule_task_scheduler_multiple():
tasks = [
ms.TuneContext(
MatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -191,6 +193,7 @@ def test_meta_schedule_task_scheduler_multiple():
),
ms.TuneContext(
MatmulReluModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -199,6 +202,7 @@ def test_meta_schedule_task_scheduler_multiple():
),
ms.TuneContext(
BatchMatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_batch_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -255,6 +259,7 @@ def
test_meta_schedule_task_scheduler_override_next_task_id_only(): # pylint: d
tasks = [
ms.TuneContext(
MatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -263,6 +268,7 @@ def
test_meta_schedule_task_scheduler_override_next_task_id_only(): # pylint: d
),
ms.TuneContext(
MatmulReluModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -271,6 +277,7 @@ def
test_meta_schedule_task_scheduler_override_next_task_id_only(): # pylint: d
),
ms.TuneContext(
BatchMatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_batch_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -310,6 +317,7 @@ def
test_meta_schedule_task_scheduler_multiple_gradient_based():
tasks = [
ms.TuneContext(
MatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -318,6 +326,7 @@ def
test_meta_schedule_task_scheduler_multiple_gradient_based():
),
ms.TuneContext(
MatmulReluModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -326,6 +335,7 @@ def
test_meta_schedule_task_scheduler_multiple_gradient_based():
),
ms.TuneContext(
BatchMatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_batch_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -394,6 +404,7 @@ def
test_meta_schedule_task_scheduler_gradient_based_with_null_search_strategy()
tasks = [
ms.TuneContext(
MatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=NullSearchStrategy(rounds_with_empty_candidates=5),
@@ -402,6 +413,7 @@ def
test_meta_schedule_task_scheduler_gradient_based_with_null_search_strategy()
),
ms.TuneContext(
BatchMatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_batch_matmul,
search_strategy=NullSearchStrategy(rounds_with_empty_candidates=0),
@@ -410,6 +422,7 @@ def
test_meta_schedule_task_scheduler_gradient_based_with_null_search_strategy()
),
ms.TuneContext(
MatmulReluModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
diff --git a/tests/python/testing/test_tvm_testing_features.py
b/tests/python/testing/test_tvm_testing_features.py
index 549aedda6a..ba2faf2a45 100644
--- a/tests/python/testing/test_tvm_testing_features.py
+++ b/tests/python/testing/test_tvm_testing_features.py
@@ -14,9 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# ruff: noqa: F401, RUF012
+# ruff: noqa: RUF012
import os
+import subprocess
import sys
import pytest
@@ -189,5 +190,44 @@ class TestPytestCache:
assert cached_fixture == param * param
+def test_empty_marker_selection_is_an_error(tmp_path):
+ test_file = tmp_path / "test_no_gpu.py"
+ test_file.write_text("def test_cpu_only():\n pass\n")
+ env = os.environ.copy()
+ env.pop("PYTEST_ADDOPTS", None)
+
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-m",
+ "pytest",
+ "-q",
+ "-p",
+ "tvm.testing.plugin",
+ "-m",
+ "gpu",
+ str(test_file),
+ ],
+ env=env,
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == pytest.ExitCode.NO_TESTS_COLLECTED,
result.stdout + result.stderr
+
+ test_file.write_text("def test_failure():\n assert False\n")
+ result = subprocess.run(
+ [sys.executable, "-m", "pytest", "-q", "-p", "tvm.testing.plugin",
str(test_file)],
+ env=env,
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == pytest.ExitCode.TESTS_FAILED, result.stdout +
result.stderr
+ assert not list(tmp_path.glob("*.xml"))
+
+
if __name__ == "__main__":
tvm.testing.main()
diff --git a/tests/scripts/task_clear_pytest.sh
b/tests/scripts/task_clear_pytest.sh
deleted file mode 100755
index 0a24f62531..0000000000
--- a/tests/scripts/task_clear_pytest.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -euxo pipefail
-
-# Ensure no stale pytest-results remain from a previous test run.
-pushd build
-rm -rf pytest-results
-popd
diff --git a/tests/scripts/task_python_nightly.sh
b/tests/scripts/task_python_nightly.sh
deleted file mode 100755
index 05070f0a04..0000000000
--- a/tests/scripts/task_python_nightly.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -euxo pipefail
-
-export PYTHONPATH="$(pwd)/python"
-export PYTEST_ADDOPTS="-s -vv ${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}"
-mkdir -p build/pytest-results
-
-# setup tvm-ffi into python folder
-uv pip install -v --target=python ./3rdparty/tvm-ffi/
-
-# cleanup pycache
-find . -type f -path "*.pyc" | xargs rm -f
-
-python3 -m pytest -n auto \
- -o junit_suite_name=python-topi-nightly \
- --junit-xml=build/pytest-results/python-topi-nightly.xml \
- --junit-prefix=cython \
- tests/python/topi/nightly
-
-# Tensorflow device verification and network tests on nightly
-export CI_ENV_NIGHTLY
-python3 tests/python/relax/test_frontend_tflite.py
diff --git a/tests/scripts/task_python_unittest.sh
b/tests/scripts/task_python_unittest.sh
index 30e5bb2bf3..c05f8503d3 100755
--- a/tests/scripts/task_python_unittest.sh
+++ b/tests/scripts/task_python_unittest.sh
@@ -20,7 +20,6 @@ set -euxo pipefail
export PYTHONPATH="$(pwd)/python"
export PYTEST_ADDOPTS="-s -vv ${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}"
-mkdir -p build/pytest-results
# cleanup pycache
find . -type f -path "*.pyc" | xargs rm -f
@@ -28,27 +27,20 @@ find . -type f -path "*.pyc" | xargs rm -f
# setup tvm-ffi into python folder
uv pip install -v --target=python ./3rdparty/tvm-ffi/
-# NOTE: also set by task_python_unittest_gpuonly.sh.
-if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then
- TVM_UNITTEST_TESTSUITE_NAME=python-unittest
-fi
-
# First run the minimal platform test. A GPU-only run can select no tests
here.
-python3 -m pytest -n auto \
- -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test"
\
-
"--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test.xml"
\
- --junit-prefix=cython \
- tests/python/all-platform-minimal-test || [ "$?" -eq 5 ]
+if [ ! -d tests/python/all-platform-minimal-test ]; then
+ echo "Missing pytest target: tests/python/all-platform-minimal-test" >&2
+ exit 1
+fi
+python3 -m pytest -n auto tests/python/all-platform-minimal-test || [ "$?" -eq
5 ]
# Then run all unit tests.
TEST_FILES=(
- "ffi"
"arith"
"ci"
"codegen"
"driver"
"ir"
- "meta_schedule"
"runtime"
"target"
"te"
@@ -57,6 +49,7 @@ TEST_FILES=(
"s_tir/schedule"
"s_tir/dlight"
"s_tir/analysis"
+ "s_tir/meta_schedule"
"s_tir/transform"
"tirx-analysis"
"tirx-base"
@@ -68,11 +61,13 @@ TEST_FILES=(
PYTEST_TARGETS=()
for TEST_FILE in "${TEST_FILES[@]}"; do
- PYTEST_TARGETS+=("tests/python/${TEST_FILE}")
+ TEST_PATH="tests/python/${TEST_FILE}"
+ if [ ! -d "${TEST_PATH}" ]; then
+ echo "Missing pytest target: ${TEST_PATH}" >&2
+ exit 1
+ fi
+ PYTEST_TARGETS+=("${TEST_PATH}")
done
-python3 -m pytest -n auto --dist=loadgroup \
- -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}" \
- "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}.xml" \
- --junit-prefix=cython \
- "${PYTEST_TARGETS[@]}"
+# Do not mask pytest's exit 5: an unexpectedly empty broad suite must fail CI.
+python3 -m pytest -n auto --dist=loadgroup "${PYTEST_TARGETS[@]}"
diff --git a/tests/scripts/task_python_unittest_gpuonly.sh
b/tests/scripts/task_python_unittest_gpuonly.sh
index 6bbd85dc10..8194b23ac4 100755
--- a/tests/scripts/task_python_unittest_gpuonly.sh
+++ b/tests/scripts/task_python_unittest_gpuonly.sh
@@ -24,21 +24,18 @@ export PYTEST_ADDOPTS="-m gpu ${PYTEST_ADDOPTS:-}"
# Test most of the enabled runtimes here.
# TODO: disabled opencl tests due to segmentation fault.
export TVM_TEST_TARGETS='cuda;metal;rocm;nvptx'
-export TVM_UNITTEST_TESTSUITE_NAME=python-unittest-gpu
./tests/scripts/task_python_unittest.sh
# Kept separate to avoid increasing time needed to run CI, testing
# only minimal functionality of Vulkan runtime.
export TVM_TEST_TARGETS='{"kind":"vulkan","from_device":0}'
-export TVM_UNITTEST_TESTSUITE_NAME=python-codegen-vulkan
export PYTHONPATH="$(pwd)/python"
export PYTEST_ADDOPTS="-s -vv ${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}"
-mkdir -p build/pytest-results
-python3 -m pytest -n auto \
- -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}" \
- "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}.xml" \
- --junit-prefix=cython \
- tests/python/codegen/test_target_codegen_vulkan.py
+if [ ! -f tests/python/codegen/test_target_codegen_vulkan.py ]; then
+ echo "Missing pytest target:
tests/python/codegen/test_target_codegen_vulkan.py" >&2
+ exit 1
+fi
+python3 -m pytest -n auto tests/python/codegen/test_target_codegen_vulkan.py
