This is an automated email from the ASF dual-hosted git repository.
areusch pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new 3e7916d30a [ci][docker] Prune all non-relevant images (#11497)
3e7916d30a is described below
commit 3e7916d30ad2b453dc6879275551b5c6a36f14cc
Author: driazati <[email protected]>
AuthorDate: Tue May 31 11:11:14 2022 -0700
[ci][docker] Prune all non-relevant images (#11497)
* [skip ci][ci][docker] Prune all non-relevant images (#11491)
Before this would leave around any image that could be used in CI. This
PR changes it so that the `docker rmi` knows exactly which image is
being used in CI so all others (even those that are being used in the
same build but not currently on that node) are deleted
This also adds some more logging so we can see what's going on and
should help keep disk usage down.
Co-authored-by: driazati <[email protected]>
* [skip ci] Revert "[skip ci][ci][docker] Prune all non-relevant images
(#11491)" (#11496)
* [ci][docker] Prune all non-relevant images
(this is a re-do of #11491)
Before this would leave around any image that could be used in CI. This PR
changes it so that the `docker rmi` knows exactly which image is being used in
CI so all others (even those that are being used in the same build but not
currently on that node) are deleted
This also adds some more logging so we can see what's going on and should
help keep disk usage down. Skipped CI since this runs during lint.
Co-authored-by: driazati <[email protected]>
---
Jenkinsfile | 88 +++++++++++++++++++++++++++++++++++++++----
jenkins/Build.groovy.j2 | 7 ++++
jenkins/DockerBuild.groovy.j2 | 8 ++++
jenkins/Lint.groovy.j2 | 1 +
jenkins/Prepare.groovy.j2 | 23 ++++++++---
jenkins/Test.groovy.j2 | 15 +++++++-
jenkins/macros.j2 | 9 +++--
7 files changed, 134 insertions(+), 17 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index d239d362f9..44389ba767 100755
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -45,7 +45,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-05-26T15:43:31.409794
+// Generated at 2022-05-27T14:45:11.226042
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// NOTE: these lines are scanned by docker/dev_common.sh. Please update the
regex as needed. -->
@@ -108,11 +108,7 @@ def per_exec_ws(folder) {
def init_git() {
checkout scm
- // Clear out all Docker images that aren't going to be used
- sh(
- script: "docker image ls --all --format '{{.Repository}}:{{.Tag}}
{{.ID}}' | { grep -vE
'${ci_arm}|${ci_cpu}|${ci_gpu}|${ci_hexagon}|${ci_i386}|${ci_lint}|${ci_qemu}|${ci_wasm}'
|| test \$? = 1; } | { xargs docker rmi || test \$? = 123; }",
- label: 'Clean old Docker images',
- )
+
// Add more info about job node
sh (
script: './tests/scripts/task_show_node_info.sh',
@@ -160,6 +156,23 @@ def init_git() {
)
}
+def docker_init(image) {
+ // Clear out all Docker images that aren't going to be used
+ sh(
+ script: """
+ set -eux
+ docker image ls --all
+ IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}}
{{.ID}}')
+
+ echo -e "Found images:\\n\$IMAGES"
+ echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs
docker rmi || test \$? = 123; }
+
+ docker image ls --all
+ """,
+ label: 'Clean old Docker images',
+ )
+}
+
def should_skip_slow_tests(pr_number) {
withCredentials([string(
credentialsId: 'tvm-bot-jenkins-reader',
@@ -321,6 +334,7 @@ def build_docker_images() {
parallel 'ci-lint': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_lint')
}
@@ -328,6 +342,7 @@ def build_docker_images() {
}, 'ci-cpu': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_cpu')
}
@@ -335,6 +350,7 @@ def build_docker_images() {
}, 'ci-gpu': {
node('GPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_gpu')
}
@@ -342,6 +358,7 @@ def build_docker_images() {
}, 'ci-qemu': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_qemu')
}
@@ -349,6 +366,7 @@ def build_docker_images() {
}, 'ci-i386': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_i386')
}
@@ -356,6 +374,7 @@ def build_docker_images() {
}, 'ci-arm': {
node('ARM') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_arm')
}
@@ -363,6 +382,7 @@ def build_docker_images() {
}, 'ci-wasm': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_wasm')
}
@@ -370,6 +390,7 @@ def build_docker_images() {
}, 'ci-hexagon': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_hexagon')
}
@@ -424,6 +445,7 @@ def lint() {
'Lint 1 of 2': {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ docker_init(ci_lint)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -441,6 +463,7 @@ def lint() {
'Lint 2 of 2': {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
+ docker_init(ci_lint)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -518,6 +541,7 @@ stage('Build') {
if (!skip_ci) {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") {
+ docker_init(ci_gpu)
init_git()
sh "${docker_run} --no-gpu ${ci_gpu}
./tests/scripts/task_config_build_gpu.sh build"
make("${ci_gpu} --no-gpu", 'build', '-j2')
@@ -564,6 +588,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") {
+ docker_init(ci_cpu)
init_git()
sh (
script: "${docker_run} ${ci_cpu}
./tests/scripts/task_config_build_cpu.sh build",
@@ -603,6 +628,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") {
+ docker_init(ci_wasm)
init_git()
sh (
script: "${docker_run} ${ci_wasm}
./tests/scripts/task_config_build_wasm.sh build",
@@ -627,6 +653,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-i386") {
+ docker_init(ci_i386)
init_git()
sh (
script: "${docker_run} ${ci_i386}
./tests/scripts/task_config_build_i386.sh build",
@@ -660,6 +687,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-arm") {
+ docker_init(ci_arm)
init_git()
sh (
script: "${docker_run} ${ci_arm}
./tests/scripts/task_config_build_arm.sh build",
@@ -691,6 +719,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-qemu") {
+ docker_init(ci_qemu)
init_git()
sh (
script: "${docker_run} ${ci_qemu}
./tests/scripts/task_config_build_qemu.sh build",
@@ -721,6 +750,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-hexagon") {
+ docker_init(ci_hexagon)
init_git()
sh (
script: "${docker_run} ${ci_hexagon}
./tests/scripts/task_config_build_hexagon.sh build",
@@ -765,6 +795,7 @@ def shard_run_unittest_GPU_1_of_3() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -830,6 +861,7 @@ def shard_run_unittest_GPU_2_of_3() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -881,6 +913,7 @@ def shard_run_unittest_GPU_3_of_3() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -929,6 +962,7 @@ def shard_run_integration_CPU_1_of_6() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
try {
+ docker_init(ci_cpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -974,6 +1008,7 @@ def shard_run_integration_CPU_2_of_6() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
try {
+ docker_init(ci_cpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1019,6 +1054,7 @@ def shard_run_integration_CPU_3_of_6() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
try {
+ docker_init(ci_cpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1064,6 +1100,7 @@ def shard_run_integration_CPU_4_of_6() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
try {
+ docker_init(ci_cpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1109,6 +1146,7 @@ def shard_run_integration_CPU_5_of_6() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
try {
+ docker_init(ci_cpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1154,6 +1192,7 @@ def shard_run_integration_CPU_6_of_6() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
try {
+ docker_init(ci_cpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1200,6 +1239,7 @@ def shard_run_python_i386_1_of_5() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
try {
+ docker_init(ci_i386)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1246,6 +1286,7 @@ def shard_run_python_i386_2_of_5() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
try {
+ docker_init(ci_i386)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1291,6 +1332,7 @@ def shard_run_python_i386_3_of_5() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
try {
+ docker_init(ci_i386)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1336,6 +1378,7 @@ def shard_run_python_i386_4_of_5() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
try {
+ docker_init(ci_i386)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1381,6 +1424,7 @@ def shard_run_python_i386_5_of_5() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
try {
+ docker_init(ci_i386)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1427,6 +1471,7 @@ def shard_run_test_Hexagon_1_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1471,6 +1516,7 @@ def shard_run_test_Hexagon_2_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1514,6 +1560,7 @@ def shard_run_test_Hexagon_3_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1557,6 +1604,7 @@ def shard_run_test_Hexagon_4_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1600,6 +1648,7 @@ def shard_run_test_Hexagon_5_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1643,6 +1692,7 @@ def shard_run_test_Hexagon_6_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1686,6 +1736,7 @@ def shard_run_test_Hexagon_7_of_7() {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
try {
+ docker_init(ci_hexagon)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1730,6 +1781,7 @@ def shard_run_integration_aarch64_1_of_4() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1774,6 +1826,7 @@ def shard_run_integration_aarch64_2_of_4() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1818,6 +1871,7 @@ def shard_run_integration_aarch64_3_of_4() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1862,6 +1916,7 @@ def shard_run_integration_aarch64_4_of_4() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1907,6 +1962,7 @@ def shard_run_topi_GPU_1_of_4() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1950,6 +2006,7 @@ def shard_run_topi_GPU_2_of_4() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -1993,6 +2050,7 @@ def shard_run_topi_GPU_3_of_4() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2036,6 +2094,7 @@ def shard_run_topi_GPU_4_of_4() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2080,6 +2139,7 @@ def shard_run_frontend_GPU_1_of_6() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2123,6 +2183,7 @@ def shard_run_frontend_GPU_2_of_6() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2166,6 +2227,7 @@ def shard_run_frontend_GPU_3_of_6() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2209,6 +2271,7 @@ def shard_run_frontend_GPU_4_of_6() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2252,6 +2315,7 @@ def shard_run_frontend_GPU_5_of_6() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2295,6 +2359,7 @@ def shard_run_frontend_GPU_6_of_6() {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
try {
+ docker_init(ci_gpu)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2339,6 +2404,7 @@ def shard_run_topi_aarch64_1_of_2() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2387,6 +2453,7 @@ def shard_run_topi_aarch64_2_of_2() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2436,6 +2503,7 @@ def shard_run_frontend_aarch64_1_of_2() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2479,6 +2547,7 @@ def shard_run_frontend_aarch64_2_of_2() {
node('ARM-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") {
try {
+ docker_init(ci_arm)
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -2648,6 +2717,7 @@ stage('Test') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") {
timeout(time: max_time, unit: 'MINUTES') {
try {
+ docker_init(ci_cpu)
init_git()
withEnv(['PLATFORM=cpu'], {
sh(
@@ -2692,6 +2762,7 @@ stage('Test') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-qemu") {
timeout(time: max_time, unit: 'MINUTES') {
try {
+ docker_init(ci_qemu)
init_git()
withEnv(['PLATFORM=qemu'], {
sh(
@@ -2736,6 +2807,7 @@ stage('Test') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-cpu") {
timeout(time: max_time, unit: 'MINUTES') {
try {
+ docker_init(ci_cpu)
init_git()
withEnv(['PLATFORM=cpu'], {
sh(
@@ -2773,6 +2845,7 @@ stage('Test') {
if (!skip_ci) {
node('GPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/docs-python-gpu") {
+ docker_init(ci_gpu)
init_git()
sh(
script: """
@@ -2814,8 +2887,7 @@ stage('Test') {
},
)
}
-}
-/*
+}/*
stage('Build packages') {
parallel 'conda CPU': {
node('CPU') {
diff --git a/jenkins/Build.groovy.j2 b/jenkins/Build.groovy.j2
index 4b0b4ae2e2..62ccc94916 100644
--- a/jenkins/Build.groovy.j2
+++ b/jenkins/Build.groovy.j2
@@ -62,6 +62,7 @@ stage('Build') {
if (!skip_ci) {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-gpu') }}) {
+ docker_init(ci_gpu)
init_git()
sh "${docker_run} --no-gpu ${ci_gpu}
./tests/scripts/task_config_build_gpu.sh build"
make("${ci_gpu} --no-gpu", 'build', '-j2')
@@ -79,6 +80,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-cpu') }}) {
+ docker_init(ci_cpu)
init_git()
sh (
script: "${docker_run} ${ci_cpu}
./tests/scripts/task_config_build_cpu.sh build",
@@ -102,6 +104,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-wasm') }}) {
+ docker_init(ci_wasm)
init_git()
sh (
script: "${docker_run} ${ci_wasm}
./tests/scripts/task_config_build_wasm.sh build",
@@ -126,6 +129,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-i386') }}) {
+ docker_init(ci_i386)
init_git()
sh (
script: "${docker_run} ${ci_i386}
./tests/scripts/task_config_build_i386.sh build",
@@ -143,6 +147,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('ARM-SMALL') {
ws({{ m.per_exec_ws('tvm/build-arm') }}) {
+ docker_init(ci_arm)
init_git()
sh (
script: "${docker_run} ${ci_arm}
./tests/scripts/task_config_build_arm.sh build",
@@ -160,6 +165,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-qemu') }}) {
+ docker_init(ci_qemu)
init_git()
sh (
script: "${docker_run} ${ci_qemu}
./tests/scripts/task_config_build_qemu.sh build",
@@ -177,6 +183,7 @@ stage('Build') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-hexagon') }}) {
+ docker_init(ci_hexagon)
init_git()
sh (
script: "${docker_run} ${ci_hexagon}
./tests/scripts/task_config_build_hexagon.sh build",
diff --git a/jenkins/DockerBuild.groovy.j2 b/jenkins/DockerBuild.groovy.j2
index 84bb8e3e37..e9d80801a9 100644
--- a/jenkins/DockerBuild.groovy.j2
+++ b/jenkins/DockerBuild.groovy.j2
@@ -59,6 +59,7 @@ def build_docker_images() {
parallel 'ci-lint': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_lint')
}
@@ -66,6 +67,7 @@ def build_docker_images() {
}, 'ci-cpu': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_cpu')
}
@@ -73,6 +75,7 @@ def build_docker_images() {
}, 'ci-gpu': {
node('GPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_gpu')
}
@@ -80,6 +83,7 @@ def build_docker_images() {
}, 'ci-qemu': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_qemu')
}
@@ -87,6 +91,7 @@ def build_docker_images() {
}, 'ci-i386': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_i386')
}
@@ -94,6 +99,7 @@ def build_docker_images() {
}, 'ci-arm': {
node('ARM') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_arm')
}
@@ -101,6 +107,7 @@ def build_docker_images() {
}, 'ci-wasm': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_wasm')
}
@@ -108,6 +115,7 @@ def build_docker_images() {
}, 'ci-hexagon': {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
+ docker_init('none')
init_git()
build_image('ci_hexagon')
}
diff --git a/jenkins/Lint.groovy.j2 b/jenkins/Lint.groovy.j2
index 61c13cd407..40dad3aef7 100644
--- a/jenkins/Lint.groovy.j2
+++ b/jenkins/Lint.groovy.j2
@@ -6,6 +6,7 @@ def lint() {
num_shards=2,
node='CPU-SMALL',
ws='tvm/lint',
+ docker_image='ci_lint',
)
%}
sh (
diff --git a/jenkins/Prepare.groovy.j2 b/jenkins/Prepare.groovy.j2
index b4db7de63b..2900775f49 100644
--- a/jenkins/Prepare.groovy.j2
+++ b/jenkins/Prepare.groovy.j2
@@ -6,11 +6,7 @@ def per_exec_ws(folder) {
def init_git() {
checkout scm
- // Clear out all Docker images that aren't going to be used
- sh(
- script: "docker image ls --all --format {% raw %}'{{.Repository}}:{{.Tag}}
{{.ID}}'{% endraw %} | { grep -vE '{% for image in images %}{% raw %}${{%
endraw %}{{ image.name }}{% raw %}}{% endraw %}{% if not loop.last %}|{% endif
%}{% endfor %}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }",
- label: 'Clean old Docker images',
- )
+
// Add more info about job node
sh (
script: './tests/scripts/task_show_node_info.sh',
@@ -58,6 +54,23 @@ def init_git() {
)
}
+def docker_init(image) {
+ // Clear out all Docker images that aren't going to be used
+ sh(
+ script: """
+ set -eux
+ docker image ls --all
+ IMAGES=\$(docker image ls --all --format {% raw
%}'{{.Repository}}:{{.Tag}} {{.ID}}'{% endraw %})
+
+ echo -e "Found images:\\n\$IMAGES"
+ echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs
docker rmi || test \$? = 123; }
+
+ docker image ls --all
+ """,
+ label: 'Clean old Docker images',
+ )
+}
+
def should_skip_slow_tests(pr_number) {
withCredentials([string(
credentialsId: 'tvm-bot-jenkins-reader',
diff --git a/jenkins/Test.groovy.j2 b/jenkins/Test.groovy.j2
index a08c50905a..9f949ae717 100644
--- a/jenkins/Test.groovy.j2
+++ b/jenkins/Test.groovy.j2
@@ -10,6 +10,7 @@
node="GPU",
ws="tvm/ut-python-gpu",
platform="gpu",
+ docker_image="ci_gpu",
test_method_names=test_method_names,
) %}
{% if shard_index == 1 %}
@@ -44,6 +45,7 @@
num_shards=6,
ws="tvm/integration-python-cpu",
platform="cpu",
+ docker_image="ci_cpu",
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }}
@@ -59,6 +61,7 @@
num_shards=5,
ws="tvm/integration-python-i386",
platform="i386",
+ docker_image="ci_i386",
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='i386', filenames=tvm_multilib) }}
@@ -78,6 +81,7 @@
node="CPU-SMALL",
ws="tvm/test-hexagon",
platform="hexagon",
+ docker_image="ci_hexagon",
test_method_names=test_method_names,
num_shards=7,
) %}
@@ -98,6 +102,7 @@
node="ARM-SMALL",
ws="tvm/ut-python-arm",
platform="arm",
+ docker_image="ci_arm",
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='arm', filenames=tvm_multilib) }}
@@ -114,6 +119,7 @@
num_shards=4,
ws="tvm/topi-python-gpu",
platform="gpu",
+ docker_image="ci_gpu",
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }}
@@ -129,6 +135,7 @@
num_shards=6,
ws="tvm/frontend-python-gpu",
platform="gpu",
+ docker_image="ci_gpu",
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='gpu', filenames=tvm_multilib) }}
@@ -143,6 +150,7 @@
node="ARM-SMALL",
ws="tvm/ut-python-arm",
platform="arm",
+ docker_image="ci_arm",
num_shards=2,
test_method_names=test_method_names,
) %}
@@ -163,6 +171,7 @@
node="ARM-SMALL",
ws="tvm/frontend-python-arm",
platform="arm",
+ docker_image="ci_arm",
num_shards=2,
test_method_names=test_method_names,
) %}
@@ -191,6 +200,7 @@ stage('Test') {
node="CPU-SMALL",
ws="tvm/ut-python-cpu",
platform="cpu",
+ docker_image="ci_cpu",
) %}
{{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }}
ci_setup(ci_cpu)
@@ -207,6 +217,7 @@ stage('Test') {
node="CPU-SMALL",
ws="tvm/test-qemu",
platform="qemu",
+ docker_image="ci_qemu",
) %}
{{ m.download_artifacts(tag='qemu', filenames=tvm_lib,
folders=microtvm_template_projects) }}
add_microtvm_permissions()
@@ -226,6 +237,7 @@ stage('Test') {
node="CPU-SMALL",
ws="tvm/frontend-python-cpu",
platform="cpu",
+ docker_image="ci_cpu",
) %}
{{ m.download_artifacts(tag='cpu', filenames=tvm_multilib) }}
ci_setup(ci_cpu)
@@ -238,6 +250,7 @@ stage('Test') {
if (!skip_ci) {
node('GPU') {
ws({{ m.per_exec_ws('tvm/docs-python-gpu') }}) {
+ docker_init(ci_gpu)
init_git()
{{ m.download_artifacts(tag='gpu', filenames=tvm_multilib,
folders=microtvm_template_projects) }}
add_microtvm_permissions()
@@ -256,4 +269,4 @@ stage('Test') {
},
)
}
-}
+}
\ No newline at end of file
diff --git a/jenkins/macros.j2 b/jenkins/macros.j2
index 1c649e31fa..5a641b73fe 100644
--- a/jenkins/macros.j2
+++ b/jenkins/macros.j2
@@ -19,7 +19,7 @@
"workspace/exec_${env.EXECUTOR_NUMBER}/{{ folder }}"
{%- endmacro -%}
-{% macro sharded_test_step(name, num_shards, node, ws, platform,
test_method_names) %}
+{% macro sharded_test_step(name, num_shards, node, ws, docker_image, platform,
test_method_names) %}
{% for shard_index in range(1, num_shards + 1) %}
{% set method_name = "shard_run_" + name.replace(":", "").replace(" ",
"-").replace("-", "_") + "_" + shard_index|string + "_of_" + num_shards|string
%}
@@ -28,6 +28,7 @@ def {{ method_name }}() {
node('{{ node }}') {
ws({{ per_exec_ws(ws) }}) {
try {
+ docker_init({{ docker_image }})
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -51,11 +52,12 @@ def {{ method_name }}() {
{% endfor %}
{% endmacro %}
-{% macro sharded_lint_step(name, num_shards, node, ws) %}
+{% macro sharded_lint_step(name, num_shards, docker_image, node, ws) %}
{% for shard_index in range(1, num_shards + 1) %}
'{{ name }} {{ shard_index }} of {{ num_shards }}': {
node('{{ node }}') {
ws({{ per_exec_ws(ws) }}) {
+ docker_init({{ docker_image }})
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
@@ -71,13 +73,14 @@ def {{ method_name }}() {
{% endmacro %}
-{% macro test_step(name, node, ws, platform) %}
+{% macro test_step(name, node, ws, docker_image, platform) %}
'{{ name }}': {
if (!skip_ci && is_docs_only_build != 1) {
node('{{ node }}') {
ws({{ per_exec_ws(ws) }}) {
timeout(time: max_time, unit: 'MINUTES') {
try {
+ docker_init({{ docker_image }})
init_git()
withEnv(['PLATFORM={{ platform }}'], {
{{ caller() | indent(width=12) | trim }}
