This is an automated email from the ASF dual-hosted git repository. areusch pushed a commit to branch areusch/freeze-dependencies in repository https://gitbox.apache.org/repos/asf/tvm.git
commit b08857c481ee3d97336268ef835fecdc030daad5 Author: Andrew Reusch <[email protected]> AuthorDate: Thu May 12 15:52:18 2022 -0700 build arch images in Jenkins --- Jenkinsfile | 153 +++++++++++++++++++++++++++++++++---------------- jenkins/Jenkinsfile.j2 | 86 +++++++++------------------ jenkins/generate.py | 15 ++++- 3 files changed, 143 insertions(+), 111 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c0fb3f5df2..adbf4d70fb 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -323,12 +323,27 @@ stage('Lint') { // a method (so the code can't all be inlined) lint() -def build_image(image_name) { +def build_base_image(arch_name) { + hash = sh( + returnStdout: true, + script: 'git log -1 --format=\'%h\'' + ).trim() + sh( + script: "docker/build-base-images.sh ${arch_name}", + label: "Build base image for ${arch_name}" + ) + archiveArtifacts artifacts: 'docker/build/base_${arch_name}/**', fingerprint: true + def files = findFiles(glob: 'docker/build/base_${arch_name}/**') + pack_lib('${arch_name}-lockfiles', files.join(', ')) +} + +def build_image(arch_name, image_name) { hash = sh( returnStdout: true, script: 'git log -1 --format=\'%h\'' ).trim() def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}" + unpack_lib('${arch_name}-lockfiles') sh( script: "${docker_build} ${image_name} --spec ${full_name}", label: 'Build docker image' @@ -380,64 +395,102 @@ def build_image(image_name) { if (rebuild_docker_images) { stage('Docker Image Build') { + // TODO in a follow up PR: Find ecr tag and use in subsequent builds - parallel 'ci-lint': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_lint') + parallel( + 'arm64': { + node('ARM') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_base_image('arm64') + } } - } - }, 'ci-cpu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_cpu') + }, + 'x86': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_base_image('x86') + } } - } - }, 'ci-gpu': { - node('GPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_gpu') + }, + 'x86_64': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_base_image('x86_64') + } } - } - }, 'ci-qemu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_qemu') + }, + ) + + parallel( + 'ci_arm': { + node('ARM') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('arm64', 'ci_arm') + } } - } - }, 'ci-i386': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_i386') + }, + 'ci_cpu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86_64', 'ci_cpu') + } } - } - }, 'ci-arm': { - node('ARM') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_arm') + }, + 'ci_gpu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86_64', 'ci_gpu') + } } - } - }, 'ci-wasm': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_wasm') + }, + 'ci_hexagon': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86_64', 'ci_hexagon') + } } - } - }, 'ci-hexagon': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_hexagon') + }, + 'ci_i386': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86', 'ci_i386') + } } - } - } + }, + 'ci_lint': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86_64', 'ci_lint') + } + } + }, + 'ci_qemu': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86_64', 'ci_qemu') + } + } + }, + 'ci_wasm': { + node('CPU') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('x86_64', 'ci_wasm') + } + } + }, + ) + } // // TODO: Once we are able to use the built images, enable this step // // If the docker images changed, we need to run the image build before the lint diff --git a/jenkins/Jenkinsfile.j2 b/jenkins/Jenkinsfile.j2 index c7534d0132..7a9046037c 100644 --- a/jenkins/Jenkinsfile.j2 +++ b/jenkins/Jenkinsfile.j2 @@ -246,9 +246,9 @@ def build_base_image(arch_name) { script: 'git log -1 --format=\'%h\'' ).trim() sh( - script: 'docker/build-base-images.sh ${arch_name}', - label: 'Build base image for ${arch_name}' - } + script: "docker/build-base-images.sh ${arch_name}", + label: "Build base image for ${arch_name}" + ) archiveArtifacts artifacts: 'docker/build/base_${arch_name}/**', fingerprint: true def files = findFiles(glob: 'docker/build/base_${arch_name}/**') pack_lib('${arch_name}-lockfiles', files.join(', ')) @@ -312,64 +312,34 @@ def build_image(arch_name, image_name) { if (rebuild_docker_images) { stage('Docker Image Build') { + // TODO in a follow up PR: Find ecr tag and use in subsequent builds - parallel 'ci-lint': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_lint') - } - } - }, 'ci-cpu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_cpu') - } - } - }, 'ci-gpu': { - node('GPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_gpu') - } - } - }, 'ci-qemu': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_qemu') - } - } - }, 'ci-i386': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_i386') - } - } - }, 'ci-arm': { - node('ARM') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_arm') - } - } - }, 'ci-wasm': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_wasm') + parallel( + {% for arch, img in nodes_by_arch(images) %} + {{ ' ' }}'{{ arch }}': { + node('{{ img.platform }}') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_base_image('{{ arch }}') + } } - } - }, 'ci-hexagon': { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - init_git() - build_image('ci_hexagon') + }, + {% endfor %} + ) + + parallel( + {% for image in images %} + '{{ image.name }}': { + node('{{ image.platform }}') { + timeout(time: max_time, unit: 'MINUTES') { + init_git() + build_image('{{ image.arch }}', '{{ image.name }}') + } } - } - } + }, + {% endfor %} + ) + } // // TODO: Once we are able to use the built images, enable this step // // If the docker images changed, we need to run the image build before the lint diff --git a/jenkins/generate.py b/jenkins/generate.py index ba7f165925..002bd64bdf 100644 --- a/jenkins/generate.py +++ b/jenkins/generate.py @@ -34,37 +34,46 @@ data = { "images": [ { "name": "ci_arm", + "arch": "arm64", "platform": "ARM", }, { "name": "ci_cpu", + "arch": "x86_64", "platform": "CPU", }, { "name": "ci_gpu", + "arch": "x86_64", "platform": "CPU", }, { "name": "ci_hexagon", + "arch": "x86_64", "platform": "CPU", }, { "name": "ci_i386", + "arch": "x86", "platform": "CPU", }, { "name": "ci_lint", + "arch": "x86_64", "platform": "CPU", }, { "name": "ci_qemu", + "arch": "x86_64", "platform": "CPU", }, { "name": "ci_wasm", + "arch": "x86_64", "platform": "CPU", }, - ] + ], + 'nodes_by_arch': lambda images: dict(sorted([(img['arch'], img) for img in images], key=lambda k: k[0])).items() } @@ -111,10 +120,10 @@ if __name__ == "__main__": Newly generated Jenkinsfile did not match the one on disk! If you have made edits to the Jenkinsfile, move them to 'jenkins/Jenkinsfile.j2' and regenerate the Jenkinsfile from the template with - + python3 -m pip install -r jenkins/requirements.txt python3 jenkins/generate.py - + Diffed changes: """ ).strip()
