This is an automated email from the ASF dual-hosted git repository. areusch pushed a commit to branch areusch/freeze-dependencies in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 588c55b13a080ef076eef60c26604c86592e10ad Author: Andrew Reusch <[email protected]> AuthorDate: Thu May 12 15:52:18 2022 -0700 build arch images in Jenkins --- Jenkinsfile | 154 +++++++++++++++++++++++++++++++------------------ jenkins/Jenkinsfile.j2 | 80 ++++++++----------------- jenkins/generate.py | 15 ++++- 3 files changed, 133 insertions(+), 116 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5978bb69ba..d2079ad2ac 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-05-11T16:27:38.745360 +// Generated at 2022-05-12T16:09:01.853886 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -323,12 +323,27 @@ stage('Lint') { // a method (so the code can't all be inlined) lint() -def build_image(image_name) { +def build_base_image(arch_name) { + hash = sh( + returnStdout: true, + script: 'git log -1 --format=\'%h\'' + ).trim() + sh( + script: 'docker/build-base-images.sh ${arch_name}', + label: 'Build base image for ${arch_name}' + ) + archiveArtifacts artifacts: 'docker/build/base_${arch_name}/**', fingerprint: true + def files = findFiles(glob: 'docker/build/base_${arch_name}/**') + pack_lib('${arch_name}-lockfiles', files.join(', ')) +} + +def build_image(arch_name, image_name) { hash = sh( returnStdout: true, script: 'git log -1 --format=\'%h\'' ).trim() def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}" + unpack_lib('${arch_name}-lockfiles') sh( script: "${docker_build} ${image_name} --spec ${full_name}", label: 'Build docker image' @@ -380,64 +395,91 @@ def build_image(image_name) { if (rebuild_docker_images) { stage('Docker Image Build') { + // TODO in a follow up PR: Find ecr tag and use in subsequent builds - parallel 'ci-lint': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_lint') - } - } - }, 'ci-cpu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_cpu') - } - } - }, 'ci-gpu': { - node('GPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_gpu') - } - } - }, 'ci-qemu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_qemu') - } - } - }, 'ci-i386': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_i386') - } - } - }, 'ci-arm': { - node('ARM') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_arm') + parallel + 'arm64': { + node('ARM') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_base_image('arm64') + } } - } - }, 'ci-wasm': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_wasm') + }, + 'x86': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_base_image('x86') + } } - } - }, 'ci-hexagon': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_hexagon') + }, + 'x86_64': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_base_image('x86_64') + } } - } - } + }, + + parallel + 'ci_arm': { + node('ARM') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('arm64', 'ci_arm') + } + }, + 'ci_cpu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86_64', 'ci_cpu') + } + }, + 'ci_gpu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86_64', 'ci_gpu') + } + }, + 'ci_hexagon': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86_64', 'ci_hexagon') + } + }, + 'ci_i386': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86', 'ci_i386') + } + }, + 'ci_lint': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86_64', 'ci_lint') + } + }, + 'ci_qemu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86_64', 'ci_qemu') + } + }, + 'ci_wasm': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86_64', 'ci_wasm') + } + }, } // // TODO: Once we are able to use the built images, enable this step // // If the docker images changed, we need to run the image build before the lint diff --git a/jenkins/Jenkinsfile.j2 b/jenkins/Jenkinsfile.j2 index 14cd099420..d77a077246 100644 --- a/jenkins/Jenkinsfile.j2 +++ b/jenkins/Jenkinsfile.j2 @@ -248,7 +248,7 @@ def build_base_image(arch_name) { sh( script: 'docker/build-base-images.sh ${arch_name}', label: 'Build base image for ${arch_name}' - } + ) archiveArtifacts artifacts: 'docker/build/base_${arch_name}/**', fingerprint: true def files = findFiles(glob: 'docker/build/base_${arch_name}/**') pack_lib('${arch_name}-lockfiles', files.join(', ')) @@ -312,64 +312,30 @@ def build_image(arch_name, image_name) { if (rebuild_docker_images) { stage('Docker Image Build') { + // TODO in a follow up PR: Find ecr tag and use in subsequent builds - parallel 'ci-lint': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_lint') - } - } - }, 'ci-cpu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_cpu') - } - } - }, 'ci-gpu': { - node('GPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_gpu') - } - } - }, 'ci-qemu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_qemu') - } - } - }, 'ci-i386': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_i386') - } - } - }, 'ci-arm': { - node('ARM') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_arm') - } - } - }, 'ci-wasm': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_wasm') - } - } - }, 'ci-hexagon': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_hexagon') + parallel + {% for arch, img in nodes_by_arch(images) %} + '{{ arch }}': { + node('{{ img.platform }}') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_base_image('{{ arch }}') + } } - } - } + }, + {% endfor %} + + parallel + {% for image in images %} + '{{ image.name }}': { + node('{{ image.platform }}') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('{{ image.arch }}', '{{ image.name }}') + } + }, + {% endfor %} } // // TODO: Once we are able to use the built images, enable this step // // If the docker images changed, we need to run the image build before the lint diff --git a/jenkins/generate.py b/jenkins/generate.py index ba7f165925..002bd64bdf 100644 --- a/jenkins/generate.py +++ b/jenkins/generate.py @@ -34,37 +34,46 @@ data = { "images": [ { "name": "ci_arm", + "arch": "arm64", "platform": "ARM", }, { "name": "ci_cpu", + "arch": "x86_64", "platform": "CPU", }, { "name": "ci_gpu", + "arch": "x86_64", "platform": "CPU", }, { "name": "ci_hexagon", + "arch": "x86_64", "platform": "CPU", }, { "name": "ci_i386", + "arch": "x86", "platform": "CPU", }, { "name": "ci_lint", + "arch": "x86_64", "platform": "CPU", }, { "name": "ci_qemu", + "arch": "x86_64", "platform": "CPU", }, { "name": "ci_wasm", + "arch": "x86_64", "platform": "CPU", }, - ] + ], + 'nodes_by_arch': lambda images: dict(sorted([(img['arch'], img) for img in images], key=lambda k: k[0])).items() } @@ -111,10 +120,10 @@ if __name__ == "__main__": Newly generated Jenkinsfile did not match the one on disk! If you have made edits to the Jenkinsfile, move them to 'jenkins/Jenkinsfile.j2' and regenerate the Jenkinsfile from the template with - + python3 -m pip install -r jenkins/requirements.txt python3 jenkins/generate.py - + Diffed changes: """ ).strip()
