This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 68a76223e ci(github): build third-party images independently per OS
version to prevent a single OS build failure from breaking the entire build
(#2346)
68a76223e is described below
commit 68a76223e54fcbd0e8f67c38f3fae94b14a5f1d8
Author: Dan Wang <[email protected]>
AuthorDate: Tue Jan 6 20:16:01 2026 +0800
ci(github): build third-party images independently per OS version to
prevent a single OS build failure from breaking the entire build (#2346)
Currently, the third-party images used for CI are built in roughly two
stages: the first stage
builds the *src* images, and the second stage builds the *bin* images. In
each stage,
images are built for multiple OS versions. If the src image build fails for
**any** OS version,
all bin image builds are canceled.
In practice, this is not ideal. The likelihood of build failures differs
across OS versions —
for example, older OS versions are more prone to issues. A failure in a
single older OS
version can block the entire third-party image build process.
This PR improves the process by fully decoupling image builds across
different OS versions
so they do not affect each other. If a third-party image build fails for
one OS version, it can
be fixed independently without impacting the builds for other OS versions.
---
...-regular-push.yml => build-push-thirdparty.yml} | 83 ++-------
.github/workflows/thirdparty-regular-push.yml | 191 +--------------------
2 files changed, 23 insertions(+), 251 deletions(-)
diff --git a/.github/workflows/thirdparty-regular-push.yml
b/.github/workflows/build-push-thirdparty.yml
similarity index 76%
copy from .github/workflows/thirdparty-regular-push.yml
copy to .github/workflows/build-push-thirdparty.yml
index f5d5e3afe..2b068e99d 100644
--- a/.github/workflows/thirdparty-regular-push.yml
+++ b/.github/workflows/build-push-thirdparty.yml
@@ -14,42 +14,19 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
----
-name: BuildThirdpartyDockerRegularly - build and publish thirdparty every week
-on:
- push:
- branches:
- - master
- - 'v[0-9]+.*' # release branch
- - ci-test # testing branch for github action
- - '*dev' # developing branch
- paths:
- - 'docker/pegasus-build-env/**'
- - 'docker/thirdparties-src/**'
- - 'docker/thirdparties-bin/**'
- - '.github/workflows/build-push-env-docker.yml'
- - '.github/workflows/thirdparty-regular-push.yml'
- - 'thirdparty/**'
-
- # for manually triggering workflow
- workflow_dispatch:
+name: BuildThirdpartyDocker - build and publish thirdparty
- # run for every week 2am UTC+8(Beijing)
- schedule:
- - cron: '0 18 * * 1'
+on:
+ workflow_call:
+ inputs:
+ osversion:
+ required: true
+ type: string
jobs:
build_push_src_docker_images:
runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- matrix:
- osversion:
- - ubuntu1804
- - ubuntu2004
- - ubuntu2204
- - rockylinux9
steps:
- uses: actions/checkout@v4
- name: Set up QEMU
@@ -68,10 +45,10 @@ jobs:
file: ./docker/thirdparties-src/Dockerfile
push: true
tags: |
- apache/pegasus:thirdparties-src-${{ matrix.osversion }}-${{
github.ref_name }}
+ apache/pegasus:thirdparties-src-${{ inputs.osversion }}-${{
github.ref_name }}
build-args: |
GITHUB_BRANCH=${{ github.ref_name }}
- OS_VERSION=${{ matrix.osversion }}
+ OS_VERSION=${{ inputs.osversion }}
HADOOP_BIN_PATH=hadoop-bin
ZOOKEEPER_BIN_PATH=zookeeper-bin
@@ -84,14 +61,6 @@ jobs:
ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
needs: build_push_src_docker_images
- strategy:
- fail-fast: false
- matrix:
- osversion:
- - ubuntu1804
- - ubuntu2004
- - ubuntu2204
- - rockylinux9
steps:
# The glibc version on ubuntu1804 is lower than the actions/checkout@v4
required, so
# we need to force to use actions/checkout@v3.
@@ -112,10 +81,10 @@ jobs:
file: ./docker/thirdparties-bin/Dockerfile
push: true
tags: |
- apache/pegasus:thirdparties-bin-${{ matrix.osversion }}-${{
github.ref_name }}
+ apache/pegasus:thirdparties-bin-${{ inputs.osversion }}-${{
github.ref_name }}
build-args: |
GITHUB_BRANCH=${{ github.ref_name }}
- OS_VERSION=${{ matrix.osversion }}
+ OS_VERSION=${{ inputs.osversion }}
HADOOP_BIN_PATH=hadoop-bin
ZOOKEEPER_BIN_PATH=zookeeper-bin
@@ -128,14 +97,6 @@ jobs:
ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
needs: build_push_src_docker_images
- strategy:
- fail-fast: false
- matrix:
- osversion:
- - ubuntu1804
- - ubuntu2004
- - ubuntu2204
- - rockylinux9
steps:
# The glibc version on ubuntu1804 is lower than the actions/checkout@v4
required, so
# we need to force to use actions/checkout@v3.
@@ -156,10 +117,10 @@ jobs:
file: ./docker/thirdparties-bin/Dockerfile
push: true
tags: |
- apache/pegasus:thirdparties-bin-jemallc-${{ matrix.osversion
}}-${{ github.ref_name }}
+ apache/pegasus:thirdparties-bin-jemallc-${{ inputs.osversion
}}-${{ github.ref_name }}
build-args: |
GITHUB_BRANCH=${{ github.ref_name }}
- OS_VERSION=${{ matrix.osversion }}
+ OS_VERSION=${{ inputs.osversion }}
USE_JEMALLOC=ON
HADOOP_BIN_PATH=hadoop-bin
ZOOKEEPER_BIN_PATH=zookeeper-bin
@@ -167,11 +128,6 @@ jobs:
build_push_bin_test_docker_images:
runs-on: ubuntu-latest
needs: build_push_src_docker_images
- strategy:
- fail-fast: false
- matrix:
- osversion:
- - ubuntu2204
steps:
- uses: actions/checkout@v4
- name: Set up QEMU
@@ -190,10 +146,10 @@ jobs:
file: ./docker/thirdparties-bin/Dockerfile
push: true
tags: |
- apache/pegasus:thirdparties-bin-test-${{ matrix.osversion }}-${{
github.ref_name }}
+ apache/pegasus:thirdparties-bin-test-${{ inputs.osversion }}-${{
github.ref_name }}
build-args: |
GITHUB_BRANCH=${{ github.ref_name }}
- OS_VERSION=${{ matrix.osversion }}
+ OS_VERSION=${{ inputs.osversion }}
ROCKSDB_PORTABLE=1
HADOOP_BIN_PATH=hadoop-bin
ZOOKEEPER_BIN_PATH=zookeeper-bin
@@ -201,11 +157,6 @@ jobs:
build_push_bin_test_jemalloc_docker_images:
runs-on: ubuntu-latest
needs: build_push_src_docker_images
- strategy:
- fail-fast: false
- matrix:
- osversion:
- - ubuntu2204
steps:
- uses: actions/checkout@v4
- name: Set up QEMU
@@ -224,10 +175,10 @@ jobs:
file: ./docker/thirdparties-bin/Dockerfile
push: true
tags: |
- apache/pegasus:thirdparties-bin-test-jemallc-${{ matrix.osversion
}}-${{ github.ref_name }}
+ apache/pegasus:thirdparties-bin-test-jemallc-${{ inputs.osversion
}}-${{ github.ref_name }}
build-args: |
GITHUB_BRANCH=${{ github.ref_name }}
- OS_VERSION=${{ matrix.osversion }}
+ OS_VERSION=${{ inputs.osversion }}
ROCKSDB_PORTABLE=1
USE_JEMALLOC=ON
HADOOP_BIN_PATH=hadoop-bin
diff --git a/.github/workflows/thirdparty-regular-push.yml
b/.github/workflows/thirdparty-regular-push.yml
index f5d5e3afe..f2c57bc0c 100644
--- a/.github/workflows/thirdparty-regular-push.yml
+++ b/.github/workflows/thirdparty-regular-push.yml
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
---
-name: BuildThirdpartyDockerRegularly - build and publish thirdparty every week
+name: BuildThirdpartyDockerRegularly - build and publish thirdparty every week
on:
push:
@@ -40,8 +40,7 @@ on:
- cron: '0 18 * * 1'
jobs:
- build_push_src_docker_images:
- runs-on: ubuntu-latest
+ build_push_thirdparty_docker_images:
strategy:
fail-fast: false
matrix:
@@ -50,185 +49,7 @@ jobs:
- ubuntu2004
- ubuntu2204
- rockylinux9
- steps:
- - uses: actions/checkout@v4
- - name: Set up QEMU
- uses: docker/setup-qemu-action@v1
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v1
- - name: Login to DockerHub
- uses: docker/login-action@v1
- with:
- username: ${{ secrets.DOCKERHUB_USER }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: Build and push
- uses: docker/[email protected]
- with:
- context: .
- file: ./docker/thirdparties-src/Dockerfile
- push: true
- tags: |
- apache/pegasus:thirdparties-src-${{ matrix.osversion }}-${{
github.ref_name }}
- build-args: |
- GITHUB_BRANCH=${{ github.ref_name }}
- OS_VERSION=${{ matrix.osversion }}
- HADOOP_BIN_PATH=hadoop-bin
- ZOOKEEPER_BIN_PATH=zookeeper-bin
-
- build_push_bin_docker_images:
- runs-on: ubuntu-latest
- env:
- # The glibc version on ubuntu1804 is lower than the node20 required, so
- # we need to force the node version to 16.
- # See more details: https://github.com/actions/checkout/issues/1809
- ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
- ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
- needs: build_push_src_docker_images
- strategy:
- fail-fast: false
- matrix:
- osversion:
- - ubuntu1804
- - ubuntu2004
- - ubuntu2204
- - rockylinux9
- steps:
- # The glibc version on ubuntu1804 is lower than the actions/checkout@v4
required, so
- # we need to force to use actions/checkout@v3.
- - uses: actions/checkout@v3
- - name: Set up QEMU
- uses: docker/setup-qemu-action@v1
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v1
- - name: Login to DockerHub
- uses: docker/login-action@v1
- with:
- username: ${{ secrets.DOCKERHUB_USER }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: Build and push for production
- uses: docker/[email protected]
- with:
- context: .
- file: ./docker/thirdparties-bin/Dockerfile
- push: true
- tags: |
- apache/pegasus:thirdparties-bin-${{ matrix.osversion }}-${{
github.ref_name }}
- build-args: |
- GITHUB_BRANCH=${{ github.ref_name }}
- OS_VERSION=${{ matrix.osversion }}
- HADOOP_BIN_PATH=hadoop-bin
- ZOOKEEPER_BIN_PATH=zookeeper-bin
-
- build_push_bin_jemalloc_docker_images:
- runs-on: ubuntu-latest
- env:
- # The glibc version on ubuntu1804 is lower than the node20 required, so
- # we need to force the node version to 16.
- # See more details: https://github.com/actions/checkout/issues/1809
- ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
- ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
- needs: build_push_src_docker_images
- strategy:
- fail-fast: false
- matrix:
- osversion:
- - ubuntu1804
- - ubuntu2004
- - ubuntu2204
- - rockylinux9
- steps:
- # The glibc version on ubuntu1804 is lower than the actions/checkout@v4
required, so
- # we need to force to use actions/checkout@v3.
- - uses: actions/checkout@v3
- - name: Set up QEMU
- uses: docker/setup-qemu-action@v1
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v1
- - name: Login to DockerHub
- uses: docker/login-action@v1
- with:
- username: ${{ secrets.DOCKERHUB_USER }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: Build and push for production with jemalloc
- uses: docker/[email protected]
- with:
- context: .
- file: ./docker/thirdparties-bin/Dockerfile
- push: true
- tags: |
- apache/pegasus:thirdparties-bin-jemallc-${{ matrix.osversion
}}-${{ github.ref_name }}
- build-args: |
- GITHUB_BRANCH=${{ github.ref_name }}
- OS_VERSION=${{ matrix.osversion }}
- USE_JEMALLOC=ON
- HADOOP_BIN_PATH=hadoop-bin
- ZOOKEEPER_BIN_PATH=zookeeper-bin
-
- build_push_bin_test_docker_images:
- runs-on: ubuntu-latest
- needs: build_push_src_docker_images
- strategy:
- fail-fast: false
- matrix:
- osversion:
- - ubuntu2204
- steps:
- - uses: actions/checkout@v4
- - name: Set up QEMU
- uses: docker/setup-qemu-action@v1
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v1
- - name: Login to DockerHub
- uses: docker/login-action@v1
- with:
- username: ${{ secrets.DOCKERHUB_USER }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: Build and push for test
- uses: docker/[email protected]
- with:
- context: .
- file: ./docker/thirdparties-bin/Dockerfile
- push: true
- tags: |
- apache/pegasus:thirdparties-bin-test-${{ matrix.osversion }}-${{
github.ref_name }}
- build-args: |
- GITHUB_BRANCH=${{ github.ref_name }}
- OS_VERSION=${{ matrix.osversion }}
- ROCKSDB_PORTABLE=1
- HADOOP_BIN_PATH=hadoop-bin
- ZOOKEEPER_BIN_PATH=zookeeper-bin
-
- build_push_bin_test_jemalloc_docker_images:
- runs-on: ubuntu-latest
- needs: build_push_src_docker_images
- strategy:
- fail-fast: false
- matrix:
- osversion:
- - ubuntu2204
- steps:
- - uses: actions/checkout@v4
- - name: Set up QEMU
- uses: docker/setup-qemu-action@v1
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v1
- - name: Login to DockerHub
- uses: docker/login-action@v1
- with:
- username: ${{ secrets.DOCKERHUB_USER }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: Build and push for test with jemalloc
- uses: docker/[email protected]
- with:
- context: .
- file: ./docker/thirdparties-bin/Dockerfile
- push: true
- tags: |
- apache/pegasus:thirdparties-bin-test-jemallc-${{ matrix.osversion
}}-${{ github.ref_name }}
- build-args: |
- GITHUB_BRANCH=${{ github.ref_name }}
- OS_VERSION=${{ matrix.osversion }}
- ROCKSDB_PORTABLE=1
- USE_JEMALLOC=ON
- HADOOP_BIN_PATH=hadoop-bin
- ZOOKEEPER_BIN_PATH=zookeeper-bin
+ uses: "./.github/workflows/build-push-thirdparty.yml"
+ with:
+ osversion: ${{ matrix.osversion }}
+ secrets: inherit
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
