This is an automated email from the ASF dual-hosted git repository.
areusch pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new c253053541 [skip ci][ci][docker] Prune all non-relevant images (#11491)
c253053541 is described below
commit c25305354199dd9b7e29d6a92a2301638559d811
Author: driazati <[email protected]>
AuthorDate: Fri May 27 11:44:47 2022 -0700
[skip ci][ci][docker] Prune all non-relevant images (#11491)
Before this would leave around any image that could be used in CI. This
PR changes it so that the `docker rmi` knows exactly which image is
being used in CI so all others (even those that are being used in the
same build but not currently on that node) are deleted
This also adds some more logging so we can see what's going on and
should help keep disk usage down.
Co-authored-by: driazati <[email protected]>
---
Jenkinsfile | 88 +++++++++++++++++++++++++++++++++++++++----
jenkins/Build.groovy.j2 | 7 ++++
jenkins/DockerBuild.groovy.j2 | 8 ++++
jenkins/Lint.groovy.j2 | 1 +
jenkins/Prepare.groovy.j2 | 23 ++++++++---
jenkins/Test.groovy.j2 | 15 +++++++-
jenkins/macros.j2 | 9 +++--
7 files changed, 134 insertions(+), 17 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index ad04bf218e..6f434a8c81 100755
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -45,7 +45,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-05-24T17:03:03.321649
+// Generated at 2022-05-27T11:07:02.305020
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// NOTE: these lines are scanned by docker/dev_common.sh. Please update the
regex as needed. -->
@@ -108,11 +108,7 @@ def per_exec_ws(folder) {
def init_git() {
checkout scm
- // Clear out all Docker images that aren't going to be used
- sh(
- script: "docker image ls --all --format '{{.Repository}}:{{.Tag}}
{{.ID}}' | { grep -vE
'${ci_arm}|${ci_cpu}|${ci_gpu}|${ci_hexagon}|${ci_i386}|${ci_lint}|${ci_qemu}|${ci_wasm}'
|| test \$? = 1; } | { xargs docker rmi || test \$? = 123; }",
- label: 'Clean old Docker images',
- )
+
// Add more info about job node
sh (
script: './tests/scripts/task_show_node_info.sh',
@@ -143,6 +139,23 @@ def init_git() {
}
}
+def docker_init(image) {
+ // Clear out all Docker images that aren't going to be used
+ sh(
+ script: """
+ set -eux
+ docker image ls --all
+ IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}}
{{.ID}}')
+
+ echo -e "Found images:\\n\$IMAGES"
+ echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs
docker rmi || test \$? = 123; }
+
+ docker image ls --all
+ """,
+ label: 'Clean old Docker images',
+ )
+}
+
def should_skip_slow_tests(pr_number) {
withCredentials([string(
credentialsId: 'tvm-bot-jenkins-reader',
@@ -304,6 +317,7 @@ def build_docker_images() {
parallel 'ci-lint': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_lint')
}
@@ -311,6 +325,7 @@ def build_docker_images() {
}, 'ci-cpu': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_cpu')
}
@@ -318,6 +333,7 @@ def build_docker_images() {
}, 'ci-gpu': {
node('GPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_gpu')
}
@@ -325,6 +341,7 @@ def build_docker_images() {
}, 'ci-qemu': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_qemu')
}
@@ -332,6 +349,7 @@ def build_docker_images() {
}, 'ci-i386': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_i386')
}
@@ -339,6 +357,7 @@ def build_docker_images() {
}, 'ci-arm': {
node('ARM') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_arm')
}
@@ -346,6 +365,7 @@ def build_docker_images() {
}, 'ci-wasm': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_wasm')
}
@@ -353,6 +373,7 @@ def build_docker_images() {
}, 'ci-hexagon': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_hexagon')
}
@@ -407,6 +428,7 @@ def lint() {
'Lint 1 of 2': {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ docker_init(ci_lint)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -424,6 +446,7 @@ def lint() {
'Lint 2 of 2': {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ docker_init(ci_lint)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -501,6 +524,7 @@ stage('Build') {
if (!skip_ci) {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") {
+ docker_init(ci_gpu)
init_git()
sh "${docker_run} --no-gpu ${ci_gpu}
./tests/scripts/task_config_build_gpu.sh build"
make("${ci_gpu} --no-gpu", 'build', '-j2')
@@ -547,6 +571,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") {
+ docker_init(ci_cpu)
init_git()
sh (
script: "${docker_run} ${ci_cpu}
./tests/scripts/task_config_build_cpu.sh build",
@@ -586,6 +611,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") {
+ docker_init(ci_wasm)
init_git()
sh (
script: "${docker_run} ${ci_wasm}
./tests/scripts/task_config_build_wasm.sh build",
@@ -610,6 +636,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-i386") {
+ docker_init(ci_386)
init_git()
sh (
script: "${docker_run} ${ci_i386}
./tests/scripts/task_config_build_i386.sh build",
@@ -643,6 +670,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-arm") {
+ docker_init(ci_arm)
init_git()
sh (
script: "${docker_run} ${ci_arm}
./tests/scripts/task_config_build_arm.sh build",
@@ -674,6 +702,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-qemu") {
+ docker_init(ci_qemu)
init_git()
sh (
script: "${docker_run} ${ci_qemu}
./tests/scripts/task_config_build_qemu.sh build",
@@ -704,6 +733,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-hexagon") {
+ docker_init(ci_hexagon)
init_git()
sh (
script: "${docker_run} ${ci_hexagon}
./tests/scripts/task_config_build_hexagon.sh build",
@@ -748,6 +778,7 @@ def shard_run_unittest_GPU_1_of_3() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -813,6 +844,7 @@ def shard_run_unittest_GPU_2_of_3() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -864,6 +896,7 @@ def shard_run_unittest_GPU_3_of_3() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -912,6 +945,7 @@ def shard_run_integration_CPU_1_of_6() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
try {
+ docker_init(ci_cpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -957,6 +991,7 @@ def shard_run_integration_CPU_2_of_6() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
try {
+ docker_init(ci_cpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1002,6 +1037,7 @@ def shard_run_integration_CPU_3_of_6() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
try {
+ docker_init(ci_cpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1047,6 +1083,7 @@ def shard_run_integration_CPU_4_of_6() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
try {
+ docker_init(ci_cpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1092,6 +1129,7 @@ def shard_run_integration_CPU_5_of_6() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
try {
+ docker_init(ci_cpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1137,6 +1175,7 @@ def shard_run_integration_CPU_6_of_6() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
try {
+ docker_init(ci_cpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1183,6 +1222,7 @@ def shard_run_python_i386_1_of_5() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
try {
+ docker_init(ci_i386)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1229,6 +1269,7 @@ def shard_run_python_i386_2_of_5() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
try {
+ docker_init(ci_i386)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1274,6 +1315,7 @@ def shard_run_python_i386_3_of_5() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
try {
+ docker_init(ci_i386)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1319,6 +1361,7 @@ def shard_run_python_i386_4_of_5() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
try {
+ docker_init(ci_i386)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1364,6 +1407,7 @@ def shard_run_python_i386_5_of_5() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
try {
+ docker_init(ci_i386)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1410,6 +1454,7 @@ def shard_run_test_Hexagon_1_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1454,6 +1499,7 @@ def shard_run_test_Hexagon_2_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1497,6 +1543,7 @@ def shard_run_test_Hexagon_3_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1540,6 +1587,7 @@ def shard_run_test_Hexagon_4_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1583,6 +1631,7 @@ def shard_run_test_Hexagon_5_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1626,6 +1675,7 @@ def shard_run_test_Hexagon_6_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1669,6 +1719,7 @@ def shard_run_test_Hexagon_7_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1713,6 +1764,7 @@ def shard_run_integration_aarch64_1_of_4() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1757,6 +1809,7 @@ def shard_run_integration_aarch64_2_of_4() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1801,6 +1854,7 @@ def shard_run_integration_aarch64_3_of_4() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1845,6 +1899,7 @@ def shard_run_integration_aarch64_4_of_4() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1890,6 +1945,7 @@ def shard_run_topi_GPU_1_of_4() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1933,6 +1989,7 @@ def shard_run_topi_GPU_2_of_4() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1976,6 +2033,7 @@ def shard_run_topi_GPU_3_of_4() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2019,6 +2077,7 @@ def shard_run_topi_GPU_4_of_4() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2063,6 +2122,7 @@ def shard_run_frontend_GPU_1_of_6() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2106,6 +2166,7 @@ def shard_run_frontend_GPU_2_of_6() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2149,6 +2210,7 @@ def shard_run_frontend_GPU_3_of_6() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2192,6 +2254,7 @@ def shard_run_frontend_GPU_4_of_6() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2235,6 +2298,7 @@ def shard_run_frontend_GPU_5_of_6() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2278,6 +2342,7 @@ def shard_run_frontend_GPU_6_of_6() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2322,6 +2387,7 @@ def shard_run_topi_aarch64_1_of_2() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2370,6 +2436,7 @@ def shard_run_topi_aarch64_2_of_2() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2419,6 +2486,7 @@ def shard_run_frontend_aarch64_1_of_2() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2462,6 +2530,7 @@ def shard_run_frontend_aarch64_2_of_2() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2631,6 +2700,7 @@ stage('Test') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") {
timeout(time: max_time, unit: 'MINUTES') {
try {
+ docker_init(ci_cpu)
init_git()
withEnv(['PLATFORM=cpu'], {
sh(
@@ -2675,6 +2745,7 @@ stage('Test') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-qemu") {
timeout(time: max_time, unit: 'MINUTES') {
try {
+ docker_init(ci_qemu)
init_git()
withEnv(['PLATFORM=qemu'], {
sh(
@@ -2719,6 +2790,7 @@ stage('Test') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-cpu") {
timeout(time: max_time, unit: 'MINUTES') {
try {
+ docker_init(ci_cpu)
init_git()
withEnv(['PLATFORM=cpu'], {
sh(
@@ -2756,6 +2828,7 @@ stage('Test') {
if (!skip_ci) {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/docs-python-gpu") {
+ docker_init(ci_gpu)
init_git()
sh(
script: """
@@ -2797,8 +2870,7 @@ stage('Test') {
},
)
}
-}
-/*
+}/*
stage('Build packages') {
parallel 'conda CPU': {
node('CPU') {
diff --git a/jenkins/Build.groovy.j2 b/jenkins/Build.groovy.j2
index 4b0b4ae2e2..7e19ce34e7 100644
--- a/jenkins/Build.groovy.j2
+++ b/jenkins/Build.groovy.j2
@@ -62,6 +62,7 @@ stage('Build') {
if (!skip_ci) {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-gpu') }}) {
+ docker_init(ci_gpu)
init_git()
sh "${docker_run} --no-gpu ${ci_gpu}
./tests/scripts/task_config_build_gpu.sh build"
make("${ci_gpu} --no-gpu", 'build', '-j2')
@@ -79,6 +80,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-cpu') }}) {
+ docker_init(ci_cpu)
init_git()
sh (
script: "${docker_run} ${ci_cpu}
./tests/scripts/task_config_build_cpu.sh build",
@@ -102,6 +104,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-wasm') }}) {
+ docker_init(ci_wasm)
init_git()
sh (
script: "${docker_run} ${ci_wasm}
./tests/scripts/task_config_build_wasm.sh build",
@@ -126,6 +129,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-i386') }}) {
+ docker_init(ci_386)
init_git()
sh (
script: "${docker_run} ${ci_i386}
./tests/scripts/task_config_build_i386.sh build",
@@ -143,6 +147,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('ARM-SMALL') {
ws({{ m.per_exec_ws('tvm/build-arm') }}) {
+ docker_init(ci_arm)
init_git()
sh (
script: "${docker_run} ${ci_arm}
./tests/scripts/task_config_build_arm.sh build",
@@ -160,6 +165,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-qemu') }}) {
+ docker_init(ci_qemu)
init_git()
sh (
script: "${docker_run} ${ci_qemu}
./tests/scripts/task_config_build_qemu.sh build",
@@ -177,6 +183,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-hexagon') }}) {
+ docker_init(ci_hexagon)
init_git()
sh (
script: "${docker_run} ${ci_hexagon}
./tests/scripts/task_config_build_hexagon.sh build",
diff --git a/jenkins/DockerBuild.groovy.j2 b/jenkins/DockerBuild.groovy.j2
index 84bb8e3e37..e9d80801a9 100644
--- a/jenkins/DockerBuild.groovy.j2
+++ b/jenkins/DockerBuild.groovy.j2
@@ -59,6 +59,7 @@ def build_docker_images() {
parallel 'ci-lint': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_lint')
}
@@ -66,6 +67,7 @@ def build_docker_images() {
}, 'ci-cpu': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_cpu')
}
@@ -73,6 +75,7 @@ def build_docker_images() {
}, 'ci-gpu': {
node('GPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_gpu')
}
@@ -80,6 +83,7 @@ def build_docker_images() {
}, 'ci-qemu': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_qemu')
}
@@ -87,6 +91,7 @@ def build_docker_images() {
}, 'ci-i386': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_i386')
}
@@ -94,6 +99,7 @@ def build_docker_images() {
}, 'ci-arm': {
node('ARM') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_arm')
}
@@ -101,6 +107,7 @@ def build_docker_images() {
}, 'ci-wasm': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_wasm')
}
@@ -108,6 +115,7 @@ def build_docker_images() {
}, 'ci-hexagon': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_hexagon')
}
diff --git a/jenkins/Lint.groovy.j2 b/jenkins/Lint.groovy.j2
index 61c13cd407..40dad3aef7 100644
--- a/jenkins/Lint.groovy.j2
+++ b/jenkins/Lint.groovy.j2
@@ -6,6 +6,7 @@ def lint() {
num_shards=2,
node='CPU-SMALL',
ws='tvm/lint',
+ docker_image='ci_lint',
)
%}
sh (
diff --git a/jenkins/Prepare.groovy.j2 b/jenkins/Prepare.groovy.j2
index d7bf5e706b..2293c2b0a6 100644
--- a/jenkins/Prepare.groovy.j2
+++ b/jenkins/Prepare.groovy.j2
@@ -6,11 +6,7 @@ def per_exec_ws(folder) {
def init_git() {
checkout scm
- // Clear out all Docker images that aren't going to be used
- sh(
- script: "docker image ls --all --format {% raw %}'{{.Repository}}:{{.Tag}}
{{.ID}}'{% endraw %} | { grep -vE '{% for image in images %}{% raw %}${{%
endraw %}{{ image.name }}{% raw %}}{% endraw %}{% if not loop.last %}|{% endif
%}{% endfor %}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }",
- label: 'Clean old Docker images',
- )
+
// Add more info about job node
sh (
script: './tests/scripts/task_show_node_info.sh',
@@ -41,6 +37,23 @@ def init_git() {
}
}
+def docker_init(image) {
+ // Clear out all Docker images that aren't going to be used
+ sh(
+ script: """
+ set -eux
+ docker image ls --all
+ IMAGES=\$(docker image ls --all --format {% raw
%}'{{.Repository}}:{{.Tag}} {{.ID}}'{% endraw %})
+
+ echo -e "Found images:\\n\$IMAGES"
+ echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs
docker rmi || test \$? = 123; }
+
+ docker image ls --all
+ """,
+ label: 'Clean old Docker images',
+ )
+}
+
def should_skip_slow_tests(pr_number) {
withCredentials([string(
credentialsId: 'tvm-bot-jenkins-reader',
diff --git a/jenkins/Test.groovy.j2 b/jenkins/Test.groovy.j2
index a08c50905a..9f949ae717 100644
--- a/jenkins/Test.groovy.j2
+++ b/jenkins/Test.groovy.j2
@@ -10,6 +10,7 @@
node="GPU",
ws="tvm/ut-python-gpu",
platform="gpu",
+ docker_image="ci_gpu",
test_method_names=test_method_names,
) %}
{% if shard_index == 1 %}
@@ -44,6 +45,7 @@
num_shards=6,
ws="tvm/integration-python-cpu",
platform="cpu",
+ docker_image="ci_cpu",
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }}
@@ -59,6 +61,7 @@
num_shards=5,
ws="tvm/integration-python-i386",
platform="i386",
+ docker_image="ci_i386",
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='i386', filenames=tvm_multilib) }}
@@ -78,6 +81,7 @@
node="CPU-SMALL",
ws="tvm/test-hexagon",
platform="hexagon",
+ docker_image="ci_hexagon",
test_method_names=test_method_names,
num_shards=7,
) %}
@@ -98,6 +102,7 @@
node="ARM-SMALL",
ws="tvm/ut-python-arm",
platform="arm",
+ docker_image="ci_arm",
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }}
@@ -114,6 +119,7 @@
num_shards=4,
ws="tvm/topi-python-gpu",
platform="gpu",
+ docker_image="ci_gpu",
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }}
@@ -129,6 +135,7 @@
num_shards=6,
ws="tvm/frontend-python-gpu",
platform="gpu",
+ docker_image="ci_gpu",
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }}
@@ -143,6 +150,7 @@
node="ARM-SMALL",
ws="tvm/ut-python-arm",
platform="arm",
+ docker_image="ci_arm",
num_shards=2,
test_method_names=test_method_names,
) %}
@@ -163,6 +171,7 @@
node="ARM-SMALL",
ws="tvm/frontend-python-arm",
platform="arm",
+ docker_image="ci_arm",
num_shards=2,
test_method_names=test_method_names,
) %}
@@ -191,6 +200,7 @@ stage('Test') {
node="CPU-SMALL",
ws="tvm/ut-python-cpu",
platform="cpu",
+ docker_image="ci_cpu",
) %}
{{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }}
ci_setup(ci_cpu)
@@ -207,6 +217,7 @@ stage('Test') {
node="CPU-SMALL",
ws="tvm/test-qemu",
platform="qemu",
+ docker_image="ci_qemu",
) %}
{{ m.download_artifacts(tag='qemu', filenames=tvm_lib,
folders=microtvm_template_projects) }}
add_microtvm_permissions()
@@ -226,6 +237,7 @@ stage('Test') {
node="CPU-SMALL",
ws="tvm/frontend-python-cpu",
platform="cpu",
+ docker_image="ci_cpu",
) %}
{{ m.download_artifacts(tag='cpu', filenames=tvm_multilib) }}
ci_setup(ci_cpu)
@@ -238,6 +250,7 @@ stage('Test') {
if (!skip_ci) {
node('GPU') {
ws({{ m.per_exec_ws('tvm/docs-python-gpu') }}) {
+ docker_init(ci_gpu)
init_git()
{{ m.download_artifacts(tag='gpu', filenames=tvm_multilib,
folders=microtvm_template_projects) }}
add_microtvm_permissions()
@@ -256,4 +269,4 @@ stage('Test') {
},
)
}
-}
+}
\ No newline at end of file
diff --git a/jenkins/macros.j2 b/jenkins/macros.j2
index 1c649e31fa..5a641b73fe 100644
--- a/jenkins/macros.j2
+++ b/jenkins/macros.j2
@@ -19,7 +19,7 @@
"workspace/exec_${env.EXECUTOR_NUMBER}/{{ folder }}"
{%- endmacro -%}
-{% macro sharded_test_step(name, num_shards, node, ws, platform,
test_method_names) %}
+{% macro sharded_test_step(name, num_shards, node, ws, docker_image, platform,
test_method_names) %}
{% for shard_index in range(1, num_shards + 1) %}
{% set method_name = "shard_run_" + name.replace(":", "").replace(" ",
"-").replace("-", "_") + "_" + shard_index|string + "_of_" + num_shards|string
%}
@@ -28,6 +28,7 @@ def {{ method_name }}() {
node('{{ node }}') {
ws({{ per_exec_ws(ws) }}) {
try {
+ docker_init({{ docker_image }})
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -51,11 +52,12 @@ def {{ method_name }}() {
{% endfor %}
{% endmacro %}
-{% macro sharded_lint_step(name, num_shards, node, ws) %}
+{% macro sharded_lint_step(name, num_shards, docker_image, node, ws) %}
{% for shard_index in range(1, num_shards + 1) %}
'{{ name }} {{ shard_index }} of {{ num_shards }}': {
node('{{ node }}') {
ws({{ per_exec_ws(ws) }}) {
+ docker_init({{ docker_image }})
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -71,13 +73,14 @@ def {{ method_name }}() {
{% endmacro %}
-{% macro test_step(name, node, ws, platform) %}
+{% macro test_step(name, node, ws, docker_image, platform) %}
'{{ name }}': {
if (!skip_ci && is_docs_only_build != 1) {
node('{{ node }}') {
ws({{ per_exec_ws(ws) }}) {
timeout(time: max_time, unit: 'MINUTES') {
try {
+ docker_init({{ docker_image }})
init_git()
withEnv(['PLATFORM={{ platform }}'], {
{{ caller() | indent(width=12) | trim }}
