This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0362ad369d Support multi-arch pinot docker image (#10429)
0362ad369d is described below
commit 0362ad369db15b696b733038d16f3a129585b660
Author: Xiang Fu <[email protected]>
AuthorDate: Wed Mar 15 19:26:24 2023 -0700
Support multi-arch pinot docker image (#10429)
---
.../build-multi-arch-pinot-docker-image.yml | 108 +++++++++++++++++++++
.../workflows/build-pinot-base-docker-image.yml | 50 ++++++++++
.../.pinot_base_docker_image_build_and_push.sh | 37 +++++++
.../scripts/docker/.pinot_build_info_gen.sh | 41 ++++++++
...not_multi_arch_docker_image_manifest_package.sh | 62 ++++++++++++
.../.pinot_single_platform_docker_image_build.sh | 66 +++++++++++++
6 files changed, 364 insertions(+)
diff --git a/.github/workflows/build-multi-arch-pinot-docker-image.yml
b/.github/workflows/build-multi-arch-pinot-docker-image.yml
new file mode 100644
index 0000000000..d6f41d059d
--- /dev/null
+++ b/.github/workflows/build-multi-arch-pinot-docker-image.yml
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Pinot Multi-Arch Platform Docker Image Build and Publish
+
+on:
+ workflow_dispatch:
+ inputs:
+ gitUrl:
+ description: "The Pinot git repo to check out to build, use https."
+ default: "https://github.com/apache/pinot.git";
+ required: true
+ commit:
+ description: "The branch/commit to check out to build Pinot image."
+ default: "master"
+ required: true
+ dockerImageName:
+ description: "The docker image name, default to 'apachepinot/pinot'."
+ default: "apachepinot/pinot"
+ required: true
+ tags:
+ description: "Tags to push of the image, comma separated, e.g.
tag1,tag2,tag3"
+ default: ""
+
+jobs:
+ generate-build-info:
+ name: Generate Build Info
+ runs-on: ubuntu-latest
+ outputs:
+ commit-id: ${{ steps.generate-build-info.outputs.commit-id }}
+ tags: ${{ steps.generate-build-info.outputs.tags }}
+ steps:
+ - uses: actions/checkout@v3
+ - name: Generate Build Info
+ id: generate-build-info
+ env:
+ PINOT_GIT_URL: ${{ github.event.inputs.gitUrl }}
+ PINOT_BRANCH: ${{ github.event.inputs.commit }}
+ TAGS: ${{ github.event.inputs.tags }}
+ run: |
+ .github/workflows/scripts/docker/.pinot_build_info_gen.sh
+ build-pinot-docker-image:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ arch: [ "amd64", "arm64" ]
+ base-image-tag: [ "11-amazoncorretto", "11-ms-openjdk" ]
+ name: Build Pinot Docker Image on ${{ matrix.arch }} with base image ${{
matrix.base-image-tag }}
+ needs: [ generate-build-info ]
+ steps:
+ - name: Login to DockerHub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - uses: docker/setup-qemu-action@v2
+ name: Set up QEMU
+ with:
+ platforms: linux/${{ matrix.arch }}
+ - uses: docker/setup-buildx-action@v2
+ name: Set up Docker Buildx
+ - uses: actions/checkout@v3
+ - name: Build and push the Docker image
+ env:
+ DOCKER_FILE_BASE_DIR: "docker/images/pinot"
+ DOCKER_IMAGE_NAME: ${{ github.event.inputs.dockerImageName }}
+ BUILD_PLATFORM: "linux/${{ matrix.arch }}"
+ BASE_IMAGE_TAG: ${{ matrix.base-image-tag }}
+ PINOT_GIT_URL: ${{ github.event.inputs.gitUrl }}
+ PINOT_BRANCH: "${{needs.generate-build-info.outputs.commit-id}}"
+ TAGS: "${{needs.generate-build-info.outputs.tags}}"
+ run:
.github/workflows/scripts/docker/.pinot_single_platform_docker_image_build.sh
+ create-multi-arch-manifest:
+ name: Create Multi-Arch Manifest
+ runs-on: ubuntu-latest
+ needs: [ generate-build-info, build-pinot-docker-image ]
+ steps:
+ - name: Login to DockerHub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - uses: docker/setup-qemu-action@v2
+ name: Set up QEMU
+ - uses: docker/setup-buildx-action@v2
+ name: Set up Docker Buildx
+ - uses: actions/checkout@v3
+ - name: Create Multi-Arch Manifest
+ env:
+ TAGS: "${{needs.generate-build-info.outputs.tags}}"
+ BUILD_PLATFORM: "linux/arm64,linux/amd64"
+ BASE_IMAGE_TAGS: "11-amazoncorretto,11-ms-openjdk"
+ run:
.github/workflows/scripts/docker/.pinot_multi_arch_docker_image_manifest_package.sh
diff --git a/.github/workflows/build-pinot-base-docker-image.yml
b/.github/workflows/build-pinot-base-docker-image.yml
new file mode 100644
index 0000000000..b540ffd61e
--- /dev/null
+++ b/.github/workflows/build-pinot-base-docker-image.yml
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+name: Pinot Base Docker Image Build and Publish
+
+on:
+ workflow_dispatch:
+ inputs: { }
+
+jobs:
+ build-pinot-build-docker-image:
+ name: Build Pinot Base Docker Image
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ baseImageType: [ "build", "runtime" ]
+ openJdkDist: [ "amazoncorretto", "ms-openjdk" ]
+ steps:
+ - name: Login to DockerHub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ - uses: docker/setup-qemu-action@v2
+ name: Set up QEMU
+ - uses: docker/setup-buildx-action@v2
+ name: Set up Docker Buildx
+ - uses: actions/checkout@v3
+ - name: Build and push the Docker image
+ env:
+ OPEN_JDK_DIST: ${{ matrix.openJdkDist }}
+ BASE_IMAGE_TYPE: ${{ matrix.baseImageType }}
+ BUILD_PLATFORM: "linux/amd64,linux/arm64"
+ TAG: "11-${{ matrix.openJdkDist }}"
+ run:
.github/workflows/scripts/docker/.pinot_base_docker_image_build_and_push.sh
diff --git
a/.github/workflows/scripts/docker/.pinot_base_docker_image_build_and_push.sh
b/.github/workflows/scripts/docker/.pinot_base_docker_image_build_and_push.sh
new file mode 100755
index 0000000000..b13b14d786
--- /dev/null
+++
b/.github/workflows/scripts/docker/.pinot_base_docker_image_build_and_push.sh
@@ -0,0 +1,37 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${BUILD_PLATFORM}" ]; then
+ exit 1
+fi
+
+if [ -z "${BASE_IMAGE_TYPE}" ]; then
+ exit 1
+fi
+
+cd docker/images/pinot-base/pinot-base-${BASE_IMAGE_TYPE}
+
+docker buildx build \
+ --no-cache \
+ --platform=${BUILD_PLATFORM} \
+ --file ${OPEN_JDK_DIST}.dockerfile \
+ --tag apachepinot/pinot-base-${BASE_IMAGE_TYPE}:${TAG} \
+ --push \
+ .
diff --git a/.github/workflows/scripts/docker/.pinot_build_info_gen.sh
b/.github/workflows/scripts/docker/.pinot_build_info_gen.sh
new file mode 100755
index 0000000000..bca2a0ef8e
--- /dev/null
+++ b/.github/workflows/scripts/docker/.pinot_build_info_gen.sh
@@ -0,0 +1,41 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+if [ -z "${PINOT_GIT_URL}" ]; then
+ PINOT_GIT_URL="https://github.com/apache/pinot.git";
+fi
+if [ -z "${PINOT_BRANCH}" ]; then
+ PINOT_BRANCH="master"
+fi
+
+# Get pinot commit id
+ROOT_DIR=$(pwd)
+rm -rf /tmp/pinot
+git clone -b ${PINOT_BRANCH} --single-branch ${PINOT_GIT_URL} /tmp/pinot
+cd /tmp/pinot
+COMMIT_ID=$(git rev-parse --short HEAD)
+VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+rm -rf /tmp/pinot
+DATE=$(date +%Y%m%d)
+
+if [ -z "${TAGS}" ]; then
+ TAGS="${VERSION}-${COMMIT_ID}-${DATE},latest"
+fi
+echo "commit-id=${COMMIT_ID}" >>"$GITHUB_OUTPUT"
+echo "tags=${TAGS}" >>"$GITHUB_OUTPUT"
diff --git
a/.github/workflows/scripts/docker/.pinot_multi_arch_docker_image_manifest_package.sh
b/.github/workflows/scripts/docker/.pinot_multi_arch_docker_image_manifest_package.sh
new file mode 100755
index 0000000000..3a2995facf
--- /dev/null
+++
b/.github/workflows/scripts/docker/.pinot_multi_arch_docker_image_manifest_package.sh
@@ -0,0 +1,62 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${DOCKER_IMAGE_NAME}" ]; then
+ DOCKER_IMAGE_NAME="apachepinot/pinot"
+fi
+
+tags=()
+declare -a tags=($(echo ${TAGS} | tr "," " "))
+
+platforms=()
+declare -a platforms=($(echo ${BUILD_PLATFORM} | tr "," " "))
+
+baseImageTags=()
+declare -a baseImageTags=($(echo ${BASE_IMAGE_TAGS} | tr "," " "))
+
+for tag in "${tags[@]}"; do
+ for baseImageTag in "${baseImageTags[@]}"; do
+ DOCKER_AMEND_TAGS_CMD=""
+ for platform in "${platforms[@]}"; do
+ platformTag=${platform/\//-}
+ DOCKER_AMEND_TAGS_CMD+=" --amend
${DOCKER_IMAGE_NAME}:${tag}-${baseImageTag}-${platformTag} "
+ done
+
+ echo "Creating manifest for tag: ${tag}-${baseImageTag}"
+ docker manifest create \
+ ${DOCKER_IMAGE_NAME}:${tag}-${baseImageTag} \
+ ${DOCKER_AMEND_TAGS_CMD}
+
+ docker manifest push \
+ ${DOCKER_IMAGE_NAME}:${tag}-${baseImageTag}
+
+ if [ "${baseImageTag}" == "11-amazoncorretto" ]; then
+ if [ "${tag}" == "latest" ]; then
+ echo "Creating manifest for tag: latest"
+ docker manifest create \
+ ${DOCKER_IMAGE_NAME}:latest \
+ ${DOCKER_AMEND_TAGS_CMD}
+
+ docker manifest push \
+ ${DOCKER_IMAGE_NAME}:latest
+ fi
+ fi
+ done
+done
diff --git
a/.github/workflows/scripts/docker/.pinot_single_platform_docker_image_build.sh
b/.github/workflows/scripts/docker/.pinot_single_platform_docker_image_build.sh
new file mode 100755
index 0000000000..c2f8b288a9
--- /dev/null
+++
b/.github/workflows/scripts/docker/.pinot_single_platform_docker_image_build.sh
@@ -0,0 +1,66 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+if [ -z "${DOCKER_IMAGE_NAME}" ]; then
+ DOCKER_IMAGE_NAME="apachepinot/pinot"
+fi
+if [ -z "${PINOT_GIT_URL}" ]; then
+ PINOT_GIT_URL="https://github.com/apache/pinot.git";
+fi
+if [ -z "${JDK_VERSION}" ]; then
+ JDK_VERSION="11"
+fi
+
+tags=()
+declare -a tags=($(echo ${TAGS} | tr "," " "))
+
+cd ${DOCKER_FILE_BASE_DIR}
+platformTag=${BUILD_PLATFORM/\//-}
+DOCKER_BUILD_TAGS=""
+for tag in "${tags[@]}"; do
+ DOCKER_BUILD_TAGS+=" --tag
${DOCKER_IMAGE_NAME}:${tag}-${BASE_IMAGE_TAG}-${platformTag} "
+
+ if [ "${BASE_IMAGE_TAG}" == "11-amazoncorretto" ]; then
+ if [ "${tag}" == "latest" ]; then
+ DOCKER_BUILD_TAGS+=" --tag ${DOCKER_IMAGE_NAME}:latest-${platformTag} "
+ fi
+ fi
+done
+
+echo "Building docker image for platform: ${BUILD_PLATFORM} with tags:
${DOCKER_BUILD_TAGS}"
+docker build \
+ --no-cache \
+ --platform ${BUILD_PLATFORM} \
+ --file Dockerfile \
+ --build-arg PINOT_GIT_URL=${PINOT_GIT_URL} \
+ --build-arg PINOT_BRANCH=${PINOT_BRANCH} \
+ --build-arg JDK_VERSION=${JDK_VERSION} \
+ ${DOCKER_BUILD_TAGS} \
+ .
+
+for tag in "${tags[@]}"; do
+ docker push ${DOCKER_IMAGE_NAME}:${tag}-${BASE_IMAGE_TAG}-${platformTag}
+
+ if [ "${BASE_IMAGE_TAG}" == "11-amazoncorretto" ]; then
+ if [ "${tag}" == "latest" ]; then
+ docker push ${DOCKER_IMAGE_NAME}:${tag}-${platformTag}
+ fi
+ fi
+done
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
