This is an automated email from the ASF dual-hosted git repository.
areusch pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new 1bde845814 [ci] Use r5.large nodes for hexagon build and some tests
(#11120)
1bde845814 is described below
commit 1bde845814dd751d11659c3ba6781a6ffc4ede45
Author: driazati <[email protected]>
AuthorDate: Tue May 17 13:37:36 2022 -0700
[ci] Use r5.large nodes for hexagon build and some tests (#11120)
* PR #11314 - [ci][docker] Update images to include sccache changes
* [ci] Use r5.large nodes for less-intensive jobs
This uses the `CPU-SMALL` label for certain jobs in CI, which is backed by
r5.large instances in EC2 rather than c4.4xlarge instances which are much more
expensive
Co-authored-by: driazati <[email protected]>
---
Jenkinsfile | 22 +++++++++++-----------
jenkins/Jenkinsfile.j2 | 21 ++++++++++++---------
tests/scripts/ci.py | 14 +++++++++++---
tests/scripts/task_build.py | 12 +++++++-----
tests/scripts/task_config_build_hexagon.sh | 2 +-
tests/scripts/task_lint.sh | 3 ---
6 files changed, 42 insertions(+), 32 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index c0fb3f5df2..6fcdc3cd4a 100755
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -45,7 +45,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-05-17T10:32:14.621387
+// Generated at 2022-05-17T09:16:58.363027
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// NOTE: these lines are scanned by docker/dev_common.sh. Please update the
regex as needed. -->
@@ -688,7 +688,7 @@ stage('Build') {
},
'BUILD: Hexagon': {
if (!skip_ci && is_docs_only_build != 1) {
- node('CPU') {
+ node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-hexagon") {
init_git()
sh (
@@ -845,7 +845,7 @@ stage('Test') {
},
'unittest: CPU': {
if (!skip_ci && is_docs_only_build != 1) {
- node('CPU') {
+ node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") {
timeout(time: max_time, unit: 'MINUTES') {
try {
@@ -873,7 +873,7 @@ stage('Test') {
},
'python: i386 1 of 3': {
if (!skip_ci && is_docs_only_build != 1) {
- node('CPU') {
+ node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
try {
init_git()
@@ -904,7 +904,7 @@ stage('Test') {
},
'python: i386 2 of 3': {
if (!skip_ci && is_docs_only_build != 1) {
- node('CPU') {
+ node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
try {
init_git()
@@ -934,7 +934,7 @@ stage('Test') {
},
'python: i386 3 of 3': {
if (!skip_ci && is_docs_only_build != 1) {
- node('CPU') {
+ node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
try {
init_git()
@@ -964,7 +964,7 @@ stage('Test') {
},
'test: Hexagon 1 of 4': {
if (!skip_ci && is_docs_only_build != 1) {
- node('CPU') {
+ node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
init_git()
@@ -997,7 +997,7 @@ stage('Test') {
},
'test: Hexagon 2 of 4': {
if (!skip_ci && is_docs_only_build != 1) {
- node('CPU') {
+ node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
init_git()
@@ -1029,7 +1029,7 @@ stage('Test') {
},
'test: Hexagon 3 of 4': {
if (!skip_ci && is_docs_only_build != 1) {
- node('CPU') {
+ node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
init_git()
@@ -1061,7 +1061,7 @@ stage('Test') {
},
'test: Hexagon 4 of 4': {
if (!skip_ci && is_docs_only_build != 1) {
- node('CPU') {
+ node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
init_git()
@@ -1093,7 +1093,7 @@ stage('Test') {
},
'test: QEMU': {
if (!skip_ci && is_docs_only_build != 1) {
- node('CPU') {
+ node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-qemu") {
timeout(time: max_time, unit: 'MINUTES') {
try {
diff --git a/jenkins/Jenkinsfile.j2 b/jenkins/Jenkinsfile.j2
index 3b2ca5d711..0264a526e7 100644
--- a/jenkins/Jenkinsfile.j2
+++ b/jenkins/Jenkinsfile.j2
@@ -605,7 +605,7 @@ stage('Build') {
},
'BUILD: Hexagon': {
if (!skip_ci && is_docs_only_build != 1) {
- node('CPU') {
+ node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-hexagon') }}) {
init_git()
sh (
@@ -681,7 +681,8 @@ stage('Test') {
{% endcall %}
{% call m.test_step(
name="unittest: CPU",
- node="CPU", ws="tvm/ut-python-cpu",
+ node="CPU-SMALL",
+ ws="tvm/ut-python-cpu",
platform="cpu",
) %}
unpack_lib('cpu', tvm_multilib_tsim)
@@ -696,11 +697,11 @@ stage('Test') {
{% endcall %}
{% call(shard_index, num_shards) m.sharded_test_step(
name="python: i386",
- node="CPU",
- num_shards=3,
- ws="tvm/integration-python-i386",
- platform="i386",
- ) %}
+ node="CPU-SMALL",
+ num_shards=3,
+ ws="tvm/integration-python-i386",
+ platform="i386",
+ ) %}
unpack_lib('i386', tvm_multilib)
ci_setup(ci_i386)
{% if shard_index == 1 %}
@@ -715,7 +716,8 @@ stage('Test') {
{% endcall %}
{% call(shard_index, num_shards) m.sharded_test_step(
name="test: Hexagon",
- node="CPU", ws="tvm/test-hexagon",
+ node="CPU-SMALL",
+ ws="tvm/test-hexagon",
platform="hexagon",
num_shards=4,
) %}
@@ -735,7 +737,8 @@ stage('Test') {
{% endcall %}
{% call m.test_step(
name="test: QEMU",
- node="CPU", ws="tvm/test-qemu",
+ node="CPU-SMALL",
+ ws="tvm/test-qemu",
platform="qemu",
) %}
unpack_lib('qemu', tvm_lib)
diff --git a/tests/scripts/ci.py b/tests/scripts/ci.py
index 22e6690beb..d45c3b1ae9 100755
--- a/tests/scripts/ci.py
+++ b/tests/scripts/ci.py
@@ -561,11 +561,14 @@ def add_subparser(
return subparser
+CPP_UNITTEST = ("run c++ unitests", ["./tests/scripts/task_cpp_unittest.sh"])
+
generated = [
generate_command(
name="gpu",
help="Run GPU build and test(s)",
options={
+ "cpp": CPP_UNITTEST,
"topi": ("run topi tests",
["./tests/scripts/task_python_topi.sh"]),
"unittest": (
"run unit tests",
@@ -582,6 +585,7 @@ generated = [
name="cpu",
help="Run CPU build and test(s)",
options={
+ "cpp": CPP_UNITTEST,
"integration": (
"run integration tests",
["./tests/scripts/task_python_integration.sh"],
@@ -601,6 +605,7 @@ generated = [
name="i386",
help="Run i386 build and test(s)",
options={
+ "cpp": CPP_UNITTEST,
"integration": (
"run integration tests",
[
@@ -619,26 +624,28 @@ generated = [
name="qemu",
help="Run QEMU build and test(s)",
options={
+ "cpp": CPP_UNITTEST,
"test": (
"run microTVM tests",
[
"./tests/scripts/task_python_microtvm.sh",
"./tests/scripts/task_demo_microtvm.sh",
],
- )
+ ),
},
),
generate_command(
name="hexagon",
help="Run Hexagon build and test(s)",
options={
+ "cpp": CPP_UNITTEST,
"test": (
"run Hexagon API/Python tests",
[
"./tests/scripts/task_build_hexagon_api.sh",
"./tests/scripts/task_python_hexagon.sh",
],
- )
+ ),
},
),
generate_command(
@@ -646,13 +653,14 @@ generated = [
help="Run ARM build and test(s) (native or via QEMU on x86)",
precheck=check_arm_qemu,
options={
+ "cpp": CPP_UNITTEST,
"python": (
"run full Python tests",
[
"./tests/scripts/task_python_unittest.sh",
"./tests/scripts/task_python_arm_compute_library.sh",
],
- )
+ ),
},
),
]
diff --git a/tests/scripts/task_build.py b/tests/scripts/task_build.py
index 52b7dd421b..e4583fe6af 100755
--- a/tests/scripts/task_build.py
+++ b/tests/scripts/task_build.py
@@ -37,21 +37,22 @@ if __name__ == "__main__":
env = {"VTA_HW_PATH": str(Path(os.getcwd()) / "3rdparty" / "vta-hw")}
sccache_exe = shutil.which("sccache")
- use_sccache = sccache_exe is not None and args.sccache_bucket is not None
+ use_sccache = sccache_exe is not None
build_dir = Path(os.getcwd()) / args.build_dir
build_dir = build_dir.relative_to(REPO_ROOT)
if use_sccache:
- env["SCCACHE_BUCKET"] = args.sccache_bucket
+ if args.sccache_bucket:
+ env["SCCACHE_BUCKET"] = args.sccache_bucket
+ logging.info(f"Using sccache bucket: {args.sccache_bucket}")
+ else:
+ logging.info(f"No sccache bucket set, using local cache")
env["CXX"] = "/opt/sccache/c++"
env["CC"] = "/opt/sccache/cc"
- logging.info(f"Using sccache bucket: {args.sccache_bucket}")
else:
if sccache_exe is None:
reason = "'sccache' executable not found"
- elif args.sccache_bucket is None:
- reason = "'sccache' executable not found"
else:
reason = "<unknown>"
logging.info(f"Not using sccache, reason: {reason}")
@@ -71,6 +72,7 @@ if __name__ == "__main__":
num_cpus = max(available_cpus, 1)
sh.run("cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo ..", cwd=build_dir)
+
target = ""
if args.cmake_target:
target = args.cmake_target
diff --git a/tests/scripts/task_config_build_hexagon.sh
b/tests/scripts/task_config_build_hexagon.sh
index c298800fcd..7bce64cddb 100755
--- a/tests/scripts/task_config_build_hexagon.sh
+++ b/tests/scripts/task_config_build_hexagon.sh
@@ -29,7 +29,7 @@ echo set\(USE_RPC ON\) >> config.cmake
echo set\(USE_MICRO ON\) >> config.cmake
echo set\(USE_MICRO_STANDALONE_RUNTIME ON\) >> config.cmake
echo set\(USE_LLVM "${CLANG_LLVM_HOME}/bin/llvm-config"\) >> config.cmake
-echo set\(CMAKE_CXX_COMPILER "${CLANG_LLVM_HOME}/bin/clang++"\) >> config.cmake
+echo set\(CMAKE_CXX_COMPILER "/opt/sccache/clang++"\) >> config.cmake
echo set\(USE_HEXAGON "ON"\) >> config.cmake
echo set\(USE_HEXAGON_SDK "${HEXAGON_SDK_PATH}"\) >> config.cmake
echo set\(USE_CCACHE OFF\) >> config.cmake
diff --git a/tests/scripts/task_lint.sh b/tests/scripts/task_lint.sh
index e0c953d618..8fbba52662 100755
--- a/tests/scripts/task_lint.sh
+++ b/tests/scripts/task_lint.sh
@@ -31,9 +31,6 @@ function shard1 {
echo "Convert scripts to Python..."
tests/scripts/task_convert_scripts_to_python.sh
- # TODO: Remove this ad-hoc pip install once
https://github.com/apache/tvm/pull/11265
- # is added to the ci_lint Docker image
- python3 -m pip install --user -r jenkins/requirements.txt
echo "Check Jenkinsfile generation"
python3 jenkins/generate.py --check
