This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 34ce707 [BEAM-9541] Push all docker images from RC instead of
hard-coding them.
new 867a298 Merge pull request #13798 from ibzib/container-pull
34ce707 is described below
commit 34ce707370f2b0ff539e0ac24c84485aeb07207e
Author: Kyle Weaver <[email protected]>
AuthorDate: Fri Jan 22 17:21:02 2021 -0800
[BEAM-9541] Push all docker images from RC instead of hard-coding them.
This greatly simplifies the script. It also spares us from having to
hard-code yet another list of supported Flink versions, which I'm trying to
avoid. There is the minor downside that listing images requires them to be
present locally from build_release_candidate.sh, so we can't clean them up in
between builds. However, I don't think that was ever really necessary except to
save disk space.
---
.../src/main/scripts/build_release_candidate.sh | 10 ---
release/src/main/scripts/publish_docker_images.sh | 87 +++-------------------
.../site/content/en/contribute/release-guide.md | 4 +-
3 files changed, 13 insertions(+), 88 deletions(-)
diff --git a/release/src/main/scripts/build_release_candidate.sh
b/release/src/main/scripts/build_release_candidate.sh
index f846e82..1c618ca 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -290,16 +290,6 @@ if [[ $confirmation = "y" ]]; then
./gradlew ":runners:spark:job-server:container:dockerPush"
-Pdocker-tag="${RELEASE}_rc${RC_NUM}"
rm -rf ~/${PYTHON_ARTIFACTS_DIR}
-
- echo '-------------------Clean up images at local-----------------'
- for ver in "${PYTHON_VER[@]}"; do
- docker rmi -f
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_rc${RC_NUM}
- done
- docker rmi -f
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}java_sdk:${RELEASE}_rc${RC_NUM}
- for ver in "${FLINK_VER[@]}"; do
- docker rmi -f
"${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}flink${ver}_job_server:${RELEASE}_rc${RC_NUM}"
- done
- docker rmi -f
"${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}spark_job_server:${RELEASE}_rc${RC_NUM}"
fi
echo "[Current Step]: Update beam-site"
diff --git a/release/src/main/scripts/publish_docker_images.sh
b/release/src/main/scripts/publish_docker_images.sh
index c40e7a2..6bf013b 100755
--- a/release/src/main/scripts/publish_docker_images.sh
+++ b/release/src/main/scripts/publish_docker_images.sh
@@ -27,10 +27,6 @@ set -e
DOCKER_IMAGE_DEFAULT_REPO_ROOT=apache
DOCKER_IMAGE_DEFAULT_REPO_PREFIX=beam_
-JAVA_VER=("java8" "java11")
-PYTHON_VER=("python3.6" "python3.7" "python3.8")
-FLINK_VER=("1.8" "1.9" "1.10")
-
echo "Publish SDK docker images to Docker Hub."
echo "================Setting Up Environment Variables==========="
@@ -47,84 +43,21 @@ echo "Do you want to proceed? [y|N]"
read confirmation
if [[ $confirmation = "y" ]]; then
- echo '-------------------Tagging and Pushing Python images-----------------'
- for ver in "${PYTHON_VER[@]}"; do
- # Pull verified RC from dockerhub.
- docker pull
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
-
- # Tag with ${RELEASE} and push to dockerhub.
- docker tag
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
- docker push
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
-
- # Tag with latest and push to dockerhub.
- docker tag
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
- docker push
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
-
- # Cleanup images from local
- docker rmi -f
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
- docker rmi -f
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
- docker rmi -f
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
- done
-
- echo '-------------------Tagging and Pushing Java images-----------------'
- for ver in "${JAVA_VER[@]}"; do
- # Pull verified RC from dockerhub.
- docker pull
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
-
- # Tag with ${RELEASE} and push to dockerhub.
- docker tag
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
- docker push
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
-
- # Tag with latest and push to dockerhub.
- docker tag
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
- docker push
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
-
- # Cleanup images from local
- docker rmi -f
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_${RC_VERSION}
- docker rmi -f
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}
- docker rmi -f
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:latest
- done
-
- echo '-------------Tagging and Pushing Flink job server images-------------'
- echo "Publishing images for the following Flink versions:" "${FLINK_VER[@]}"
- echo "Make sure the versions are correct, then press any key to proceed."
+ IMAGES=$(docker images --filter
"reference=apache/beam_*:${RELEASE}_${RC_VERSION}" --format "{{.Repository}}")
+ echo "Publishing the following images:" "${IMAGES[@]}"
+ echo "Make sure the images are correct, then press any key to proceed."
read
- for ver in "${FLINK_VER[@]}"; do
-
FLINK_IMAGE_NAME=${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}flink${ver}_job_server
-
+ for IMAGE in "${IMAGES[@]}"; do
# Pull verified RC from dockerhub.
- docker pull "${FLINK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}"
+ docker pull ${IMAGE}:${RELEASE}_${RC_VERSION}
# Tag with ${RELEASE} and push to dockerhub.
- docker tag "${FLINK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}"
"${FLINK_IMAGE_NAME}:${RELEASE}"
- docker push "${FLINK_IMAGE_NAME}:${RELEASE}"
+ docker tag ${IMAGE}:${RELEASE}_${RC_VERSION} ${IMAGE}:${RELEASE}
+ docker push ${IMAGE}:${RELEASE}
# Tag with latest and push to dockerhub.
- docker tag "${FLINK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}"
"${FLINK_IMAGE_NAME}:latest"
- docker push "${FLINK_IMAGE_NAME}:latest"
-
- # Cleanup images from local
- docker rmi -f "${FLINK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}"
- docker rmi -f "${FLINK_IMAGE_NAME}:${RELEASE}"
- docker rmi -f "${FLINK_IMAGE_NAME}:latest"
+ docker tag ${IMAGE}:${RELEASE}_${RC_VERSION} ${IMAGE}:latest
+ docker push ${IMAGE}:latest
done
- echo '-------------Tagging and Pushing Spark job server image-------------'
-
SPARK_IMAGE_NAME=${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}spark_job_server
-
- # Pull verified RC from dockerhub.
- docker pull "${SPARK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}"
-
- # Tag with ${RELEASE} and push to dockerhub.
- docker tag "${SPARK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}"
"${SPARK_IMAGE_NAME}:${RELEASE}"
- docker push "${SPARK_IMAGE_NAME}:${RELEASE}"
-
- # Tag with latest and push to dockerhub.
- docker tag "${SPARK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}"
"${SPARK_IMAGE_NAME}:latest"
- docker push "${SPARK_IMAGE_NAME}:latest"
-
- # Cleanup images from local
- docker rmi -f "${SPARK_IMAGE_NAME}:${RELEASE}_${RC_VERSION}"
- docker rmi -f "${SPARK_IMAGE_NAME}:${RELEASE}"
- docker rmi -f "${SPARK_IMAGE_NAME}:latest"
-fi
+fi
\ No newline at end of file
diff --git a/website/www/site/content/en/contribute/release-guide.md
b/website/www/site/content/en/contribute/release-guide.md
index 4ef7596..6f8ab5c 100644
--- a/website/www/site/content/en/contribute/release-guide.md
+++ b/website/www/site/content/en/contribute/release-guide.md
@@ -1089,7 +1089,7 @@ please follow [the
guide](https://help.github.com/articles/creating-a-personal-a
All wheels should be published, in addition to the zip of the release source.
(Signatures and hashes do _not_ need to be uploaded.)
-### Deploy SDK docker images to DockerHub
+### Deploy docker images to DockerHub
* Script:
[publish_docker_images.sh](https://github.com/apache/beam/blob/master/release/src/main/scripts/publish_docker_images.sh)
* Usage
```
@@ -1100,6 +1100,8 @@ Verify that:
* Images with *latest* tag are pointing to current release by confirming
1. Digest of the image with *latest* tag is the same as the one with
{RELEASE} tag.
+(Optional) Clean up any unneeded local images afterward to save disk space.
+
### Merge Website pull requests
Merge all of the website pull requests
