This is an automated email from the ASF dual-hosted git repository. tqchen pushed a commit to branch jenkins-ecr-cache in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 025d4bf2a1653eac57d0d2b80f86a67572508c93 Author: Tianqi Chen <[email protected]> AuthorDate: Thu Oct 15 13:45:45 2026 +0000 [CI] Pull Jenkins Docker Hub images through the ECR cache --- ci/jenkins/README.md | 18 ++++++++ ci/jenkins/generated/common.groovy | 57 ++++++++++++++++---------- ci/jenkins/generated/docker_jenkinsfile.groovy | 2 +- 3 files changed, 55 insertions(+), 22 deletions(-) diff --git a/ci/jenkins/README.md b/ci/jenkins/README.md index 0f8062b0d9..c40bca4dd1 100644 --- a/ci/jenkins/README.md +++ b/ci/jenkins/README.md @@ -80,6 +80,24 @@ in `../github/`. The retained `determine_docker_images.py` supports both Jenkins image selection and local `--lookup-only` lookup; `should_rebuild_docker.py` and the standalone skip commands retain their command-line interfaces. +### Docker Hub pull-through cache + +Jenkins keeps Docker Hub image names in `docker-images.ini` and image job +parameters. After published/staging image selection and parameter overrides, +`common.groovy` maps Hub references to +`477529581014.dkr.ecr.us-west-2.amazonaws.com/docker-hub`. Build and test stages +pull and run that same resolved image. Official images use the `library/` +namespace; explicit non-Hub registries, tags and digests are preserved. Docker +publishing retains the original selected Hub names. + +The optional Jenkins environment setting `DOCKER_HUB_CACHE` overrides the cache +registry and repository prefix. An explicitly empty value disables translation; +unset uses the configured default, including on named branches. No controller +change is needed for the default. ECR login uses the selected image's registry +and region, retries pulls, and removes login credentials afterward. Authentication +or cache-import failures fail the job rather than falling back to Docker Hub. +Local `docker/bash.sh` and image lookup continue using public names unchanged. + ### Trust and rollout Each entry point evaluates the shared helper returned by `readTrusted` at diff --git a/ci/jenkins/generated/common.groovy b/ci/jenkins/generated/common.groovy index 0f9a14801e..b93500e0e0 100644 --- a/ci/jenkins/generated/common.groovy +++ b/ci/jenkins/generated/common.groovy @@ -20,6 +20,7 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // Keep the image names aligned with docker-images.ini. Tags are resolved in prepare. image_platforms = [ci_arm: 'ARM', ci_cpu: 'CPU', ci_gpu: 'GPU', ci_wasm: 'CPU'] images = image_platforms.collectEntries { name, platform -> [(name): ''] } +source_images = [:] built_images = [:] upstream_revision = null max_time = 180 @@ -122,18 +123,36 @@ def init_git() { checkout_trusted_files() } -def ecr_pull(full_name) { - aws_account_id = sh( - returnStdout: true, - script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"', - label: 'Get AWS ID' - ).trim() +// Apply only after Hub selection and job overrides; inputs stay public image names. +def docker_hub_image(image) { + def cache = env.DOCKER_HUB_CACHE + if (cache == null) cache = '477529581014.dkr.ecr.us-west-2.amazonaws.com/docker-hub' + cache = cache.trim().replaceAll('/+$', '') + if (!cache || !image) return image + + def path = image + if (image.contains('/')) { + def host = image.substring(0, image.indexOf('/')) + if (host in ['docker.io', 'index.docker.io', 'registry-1.docker.io']) { + path = image.substring(image.indexOf('/') + 1) + } else if (host.contains('.') || host.contains(':') || host == 'localhost') { + return image + } + } + if (!path.contains('/')) path = "library/${path}" + return "${cache}/${path}" +} - try { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2', - "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { +def ecr_registry(image) { + def registry = image.tokenize('/')[0] + return registry ==~ /[0-9]{12}\.dkr\.ecr\.[a-z0-9-]+\.amazonaws\.com(\.cn)?/ ? registry : null +} + +def ecr_pull(full_name) { + def registry = ecr_registry(full_name) + def region = registry.tokenize('.')[3] + withEnv(["AWS_DEFAULT_REGION=${region}", "AWS_ECR_REPO=${registry}"]) { + try { sh( script: ''' set -eux @@ -149,12 +168,7 @@ def ecr_pull(full_name) { """, label: 'Pull image from ECR' ) - } - } finally { - withEnv([ - "AWS_ACCOUNT_ID=${aws_account_id}", - 'AWS_DEFAULT_REGION=us-west-2', - "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) { + } finally { sh( script: 'docker logout $AWS_ECR_REPO', label: 'Clean up login credentials' @@ -231,7 +245,10 @@ def prepare(node_type) { ).trim() } } - images.keySet().each { name -> images[name] = params["${name}_param"] ?: images[name] } + images.keySet().each { name -> + source_images[name] = params["${name}_param"] ?: images[name] + images[name] = docker_hub_image(source_images[name]) + } sh( script: 'echo "Docker images being used in this build:"\n' + images.collect { name, image -> "echo \" ${name} = ${image}\"" }.join('\n'), @@ -338,9 +355,7 @@ def docker_init(image) { label: 'Clean old Docker images', ) - if (image.contains("amazonaws.com")) { - // If this string is in the image name it's from ECR and needs to be pulled - // with the right credentials + if (ecr_registry(image)) { ecr_pull(image) } else { sh( diff --git a/ci/jenkins/generated/docker_jenkinsfile.groovy b/ci/jenkins/generated/docker_jenkinsfile.groovy index 0039be7712..35da2c773f 100644 --- a/ci/jenkins/generated/docker_jenkinsfile.groovy +++ b/ci/jenkins/generated/docker_jenkinsfile.groovy @@ -129,7 +129,7 @@ def deploy() { withCredentials([string(credentialsId: 'dockerhub-tlcpack-key', variable: 'TLCPACK_TOKEN')]) { try { sh(script: 'echo $TLCPACK_TOKEN | docker login --username octomldriazati --password-stdin', label: 'Log in to Docker Hub') - images.each { name, image -> + source_images.each { name, image -> if (image.contains('tlcpackstaging')) { def tag = image.split(':')[1] sh(
