This is an automated email from the ASF dual-hosted git repository. areusch pushed a commit to branch areusch/freeze-dependencies in repository https://gitbox.apache.org/repos/asf/tvm.git
commit dc5f159b8210dbe9a485bc9a203b501fa78d7eda Author: Andrew Reusch <[email protected]> AuthorDate: Tue May 24 12:28:47 2022 -0700 fix docker images to built ones for testing. --- Jenkinsfile | 105 ++++++++++++++++++++++++++++++++++++------ jenkins/Build.groovy.j2 | 7 +++ jenkins/Deploy.groovy.j2 | 1 + jenkins/DockerBuild.groovy.j2 | 53 +++++++++++++++++++++ jenkins/Jenkinsfile.j2 | 34 +++++++++----- jenkins/Lint.groovy.j2 | 1 + jenkins/Test.groovy.j2 | 4 ++ 7 files changed, 180 insertions(+), 25 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 347d6725ab..34a24e4d5d 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,18 +45,27 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-05-24T12:25:02.041933 +// Generated at 2022-05-25T11:48:25.852378 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> -ci_lint = 'tlcpack/ci-lint:20220513-055910-fa834f67e' -ci_gpu = 'tlcpack/ci-gpu:20220519-055908-ddfa1da69' -ci_cpu = 'tlcpack/ci-cpu:20220519-055908-ddfa1da69' -ci_wasm = 'tlcpack/ci-wasm:20220513-055910-fa834f67e' -ci_i386 = 'tlcpack/ci-i386:20220513-055910-fa834f67e' -ci_qemu = 'tlcpack/ci-qemu:20220517-094028-de21c8f2e' -ci_arm = 'tlcpack/ci-arm:20220513-055910-fa834f67e' -ci_hexagon = 'tlcpack/ci-hexagon:20220516-190055-672ce3365' +def aws_account_id = '' +node("CPU-SMALL") { +aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() +} +def AWS_DEFAULT_REGION = 'us-west-2' +ci_lint = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_lint:areusch-2ffreeze-dependencies-4233ca6eb-49" +ci_gpu = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_gpu:areusch-2ffreeze-dependencies-6799bd6f9-49" +ci_cpu = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_cpu:areusch-2ffreeze-dependencies-98aa6dde6-49" +ci_wasm = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_wasm:areusch-2ffreeze-dependencies-c12a78740-49" +ci_i386 = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_i386:areusch-2ffreeze-dependencies-89e22c948-49" +ci_qemu = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_qemu:areusch-2ffreeze-dependencies-e839c36c8-48" +ci_arm = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_arm:areusch-2ffreeze-dependencies-b9d48a35f-49" +ci_hexagon = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_hexagon:areusch-2ffreeze-dependencies-e7d44544e-49" // <--- End of regex-scanned config. // Parameters to allow overriding (in Jenkins UI), the images @@ -94,6 +103,7 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) { return } + // Filenames for stashing between build and test steps s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" @@ -429,6 +439,59 @@ def build_docker_images() { // } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION=us-west-2', + "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + +def docker_init(image) { + if (image.contains("amazonaws.com")) { + // If this string is in the image name it's from ECR and needs to be pulled + // with the right credentials + ecr_pull(image) + } else { + sh( + script: "docker pull ${image}", + label: 'Pull docker image', + ) + } +} + // Run make. First try to do an incremental make from a previous workspace in hope to // accelerate the compilation. If something is wrong, clean the workspace and then // build from scratch. @@ -463,7 +526,8 @@ def lint() { withEnv([ 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=0'], { - sh ( + docker_init(ci_lint) + sh ( script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", label: 'Run lint', ) @@ -481,7 +545,8 @@ def lint() { withEnv([ 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=1'], { - sh ( + docker_init(ci_lint) + sh ( script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", label: 'Run lint', ) @@ -555,6 +620,7 @@ stage('Build') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") { docker_init(ci_gpu) init_git() + docker_init(ci_gpu) sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" make("${ci_gpu} --no-gpu", 'build', '-j2') sh( @@ -602,6 +668,7 @@ stage('Build') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") { docker_init(ci_cpu) init_git() + docker_init(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", label: 'Create CPU cmake config', @@ -642,6 +709,7 @@ stage('Build') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") { docker_init(ci_wasm) init_git() + docker_init(ci_wasm) sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", label: 'Create WASM cmake config', @@ -667,6 +735,7 @@ stage('Build') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-i386") { docker_init(ci_i386) init_git() + docker_init(ci_i386) sh ( script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build", label: 'Create i386 cmake config', @@ -701,6 +770,7 @@ stage('Build') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-arm") { docker_init(ci_arm) init_git() + docker_init(ci_arm) sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build", label: 'Create ARM cmake config', @@ -733,6 +803,7 @@ stage('Build') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-qemu") { docker_init(ci_qemu) init_git() + docker_init(ci_qemu) sh ( script: "${docker_run} ${ci_qemu} ./tests/scripts/task_config_build_qemu.sh build", label: 'Create QEMU cmake config', @@ -764,6 +835,7 @@ stage('Build') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-hexagon") { docker_init(ci_hexagon) init_git() + docker_init(ci_hexagon) sh ( script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build", label: 'Create Hexagon cmake config', @@ -3039,6 +3111,7 @@ stage('Test') { docker_init(ci_cpu) init_git() withEnv(['PLATFORM=cpu'], { + docker_init(ci_cpu) sh( script: """ set -eux @@ -3092,6 +3165,7 @@ stage('Test') { docker_init(ci_qemu) init_git() withEnv(['PLATFORM=qemu'], { + docker_init(ci_qemu) sh( script: """ set -eux @@ -3145,6 +3219,7 @@ stage('Test') { docker_init(ci_cpu) init_git() withEnv(['PLATFORM=cpu'], { + docker_init(ci_cpu) sh( script: """ set -eux @@ -3190,6 +3265,7 @@ stage('Test') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/docs-python-gpu") { docker_init(ci_gpu) init_git() + docker_init(ci_gpu) sh( script: """ set -eux @@ -3297,6 +3373,7 @@ def deploy() { if (env.BRANCH_NAME == 'main' && env.DOCS_DEPLOY_ENABLED == 'yes') { node('CPU') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/deploy-docs") { + docker_init(ci_gpu) sh( script: """ set -eux @@ -3318,9 +3395,9 @@ cancel_previous_build() prepare() -if (rebuild_docker_images) { - build_docker_images() -} +// if (rebuild_docker_images) { +// build_docker_images() +// } lint() diff --git a/jenkins/Build.groovy.j2 b/jenkins/Build.groovy.j2 index 62ccc94916..a4bb11f0cf 100644 --- a/jenkins/Build.groovy.j2 +++ b/jenkins/Build.groovy.j2 @@ -64,6 +64,7 @@ stage('Build') { ws({{ m.per_exec_ws('tvm/build-gpu') }}) { docker_init(ci_gpu) init_git() + docker_init(ci_gpu) sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" make("${ci_gpu} --no-gpu", 'build', '-j2') {{ m.upload_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }} @@ -82,6 +83,7 @@ stage('Build') { ws({{ m.per_exec_ws('tvm/build-cpu') }}) { docker_init(ci_cpu) init_git() + docker_init(ci_cpu) sh ( script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build", label: 'Create CPU cmake config', @@ -106,6 +108,7 @@ stage('Build') { ws({{ m.per_exec_ws('tvm/build-wasm') }}) { docker_init(ci_wasm) init_git() + docker_init(ci_wasm) sh ( script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build", label: 'Create WASM cmake config', @@ -131,6 +134,7 @@ stage('Build') { ws({{ m.per_exec_ws('tvm/build-i386') }}) { docker_init(ci_i386) init_git() + docker_init(ci_i386) sh ( script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build", label: 'Create i386 cmake config', @@ -149,6 +153,7 @@ stage('Build') { ws({{ m.per_exec_ws('tvm/build-arm') }}) { docker_init(ci_arm) init_git() + docker_init(ci_arm) sh ( script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build", label: 'Create ARM cmake config', @@ -167,6 +172,7 @@ stage('Build') { ws({{ m.per_exec_ws('tvm/build-qemu') }}) { docker_init(ci_qemu) init_git() + docker_init(ci_qemu) sh ( script: "${docker_run} ${ci_qemu} ./tests/scripts/task_config_build_qemu.sh build", label: 'Create QEMU cmake config', @@ -185,6 +191,7 @@ stage('Build') { ws({{ m.per_exec_ws('tvm/build-hexagon') }}) { docker_init(ci_hexagon) init_git() + docker_init(ci_hexagon) sh ( script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build", label: 'Create Hexagon cmake config', diff --git a/jenkins/Deploy.groovy.j2 b/jenkins/Deploy.groovy.j2 index 917f71ded1..02b7f5a40a 100644 --- a/jenkins/Deploy.groovy.j2 +++ b/jenkins/Deploy.groovy.j2 @@ -62,6 +62,7 @@ def deploy() { if (env.BRANCH_NAME == 'main' && env.DOCS_DEPLOY_ENABLED == 'yes') { node('CPU') { ws({{ m.per_exec_ws('tvm/deploy-docs') }}) { + docker_init(ci_gpu) {{ m.download_artifacts(tag='docs', filenames=["docs.tgz"]) }} deploy_docs() } diff --git a/jenkins/DockerBuild.groovy.j2 b/jenkins/DockerBuild.groovy.j2 index 68cbe49413..c7c1a3acdb 100644 --- a/jenkins/DockerBuild.groovy.j2 +++ b/jenkins/DockerBuild.groovy.j2 @@ -154,6 +154,59 @@ def build_docker_images() { // } } +def ecr_pull(full_name) { + aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() + + try { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION={{ aws_default_region }}', + "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) { + sh( + script: ''' + set -eux + aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO + ''', + label: 'Log in to ECR' + ) + sh( + script: """ + set -eux + docker pull ${full_name} + """, + label: 'Pull image from ECR' + ) + } + } finally { + withEnv([ + "AWS_ACCOUNT_ID=${aws_account_id}", + 'AWS_DEFAULT_REGION={{ aws_default_region }}', + "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) { + sh( + script: 'docker logout $AWS_ECR_REPO', + label: 'Clean up login credentials' + ) + } + } +} + +def docker_init(image) { + if (image.contains("amazonaws.com")) { + // If this string is in the image name it's from ECR and needs to be pulled + // with the right credentials + ecr_pull(image) + } else { + sh( + script: "docker pull ${image}", + label: 'Pull docker image', + ) + } +} + // Run make. First try to do an incremental make from a previous workspace in hope to // accelerate the compilation. If something is wrong, clean the workspace and then // build from scratch. diff --git a/jenkins/Jenkinsfile.j2 b/jenkins/Jenkinsfile.j2 index c165de964f..e90f161858 100644 --- a/jenkins/Jenkinsfile.j2 +++ b/jenkins/Jenkinsfile.j2 @@ -51,14 +51,23 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils {% import 'jenkins/macros.j2' as m with context -%} // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> -ci_lint = 'tlcpack/ci-lint:20220513-055910-fa834f67e' -ci_gpu = 'tlcpack/ci-gpu:20220519-055908-ddfa1da69' -ci_cpu = 'tlcpack/ci-cpu:20220519-055908-ddfa1da69' -ci_wasm = 'tlcpack/ci-wasm:20220513-055910-fa834f67e' -ci_i386 = 'tlcpack/ci-i386:20220513-055910-fa834f67e' -ci_qemu = 'tlcpack/ci-qemu:20220517-094028-de21c8f2e' -ci_arm = 'tlcpack/ci-arm:20220513-055910-fa834f67e' -ci_hexagon = 'tlcpack/ci-hexagon:20220516-190055-672ce3365' +def aws_account_id = '' +node("CPU-SMALL") { +aws_account_id = sh( + returnStdout: true, + script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', + label: 'Get AWS ID' + ).trim() +} +def AWS_DEFAULT_REGION = 'us-west-2' +ci_lint = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_lint:areusch-2ffreeze-dependencies-4233ca6eb-49" +ci_gpu = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_gpu:areusch-2ffreeze-dependencies-6799bd6f9-49" +ci_cpu = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_cpu:areusch-2ffreeze-dependencies-98aa6dde6-49" +ci_wasm = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_wasm:areusch-2ffreeze-dependencies-c12a78740-49" +ci_i386 = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_i386:areusch-2ffreeze-dependencies-89e22c948-49" +ci_qemu = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_qemu:areusch-2ffreeze-dependencies-e839c36c8-48" +ci_arm = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_arm:areusch-2ffreeze-dependencies-b9d48a35f-49" +ci_hexagon = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_hexagon:areusch-2ffreeze-dependencies-e7d44544e-49" // <--- End of regex-scanned config. // Parameters to allow overriding (in Jenkins UI), the images @@ -91,6 +100,9 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) { return } +{% set aws_default_region = "us-west-2" %} +{% set aws_ecr_url = "dkr.ecr." + aws_default_region + ".amazonaws.com" %} + // Filenames for stashing between build and test steps {% set tvm_runtime = ['build/libtvm_runtime.so', 'build/config.cmake'] %} {% set tvm_lib = ['build/libtvm.so'] + tvm_runtime %} @@ -115,9 +127,9 @@ cancel_previous_build() prepare() -if (rebuild_docker_images) { - build_docker_images() -} +// if (rebuild_docker_images) { +// build_docker_images() +// } lint() diff --git a/jenkins/Lint.groovy.j2 b/jenkins/Lint.groovy.j2 index 40dad3aef7..717ba79348 100644 --- a/jenkins/Lint.groovy.j2 +++ b/jenkins/Lint.groovy.j2 @@ -9,6 +9,7 @@ def lint() { docker_image='ci_lint', ) %} + docker_init(ci_lint) sh ( script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", label: 'Run lint', diff --git a/jenkins/Test.groovy.j2 b/jenkins/Test.groovy.j2 index d219b47bc7..393ed6243a 100644 --- a/jenkins/Test.groovy.j2 +++ b/jenkins/Test.groovy.j2 @@ -206,6 +206,7 @@ stage('Test') { platform="cpu", docker_image="ci_cpu", ) %} + docker_init(ci_cpu) {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }} ci_setup(ci_cpu) cpp_unittest(ci_cpu) @@ -223,6 +224,7 @@ stage('Test') { platform="qemu", docker_image="ci_qemu", ) %} + docker_init(ci_qemu) {{ m.download_artifacts(tag='qemu', filenames=tvm_lib, folders=microtvm_template_projects) }} add_microtvm_permissions() ci_setup(ci_qemu) @@ -243,6 +245,7 @@ stage('Test') { platform="cpu", docker_image="ci_cpu", ) %} + docker_init(ci_cpu) {{ m.download_artifacts(tag='cpu', filenames=tvm_multilib) }} ci_setup(ci_cpu) sh ( @@ -256,6 +259,7 @@ stage('Test') { ws({{ m.per_exec_ws('tvm/docs-python-gpu') }}) { docker_init(ci_gpu) init_git() + docker_init(ci_gpu) {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }} add_microtvm_permissions() timeout(time: 180, unit: 'MINUTES') {
