This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/main by this push:
new 53796c7a60c [ci] Trigger Docker images independently (#415)
53796c7a60c is described below
commit 53796c7a60c40f0e089392f37ac5ceea741542ad
Author: Dongyang Li <[email protected]>
AuthorDate: Mon Sep 14 19:36:44 2026 +0800
[ci] Trigger Docker images independently (#415)
---
.github/workflows/build.yml | 100 ++++++++++++++++++++++++++------------------
1 file changed, 59 insertions(+), 41 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 6bb96c5b2ba..278142ceab0 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -24,6 +24,10 @@ on:
description: "Force run build job when manually triggered"
required: false
default: "true"
+ docker_only:
+ description: "Only rebuild and push Docker images"
+ required: false
+ default: "false"
schedule:
- cron: '*/30 * * * *'
@@ -49,6 +53,7 @@ jobs:
contents: write
outputs:
should_release: ${{ steps.check_diff.outputs.should_release }}
+ should_update_docker: ${{ steps.check_diff.outputs.should_update_docker
}}
should_build_macos_x86: ${{
steps.check_diff.outputs.should_build_macos_x86 }}
thirdparty_commit_hash: ${{
steps.check_diff.outputs.thirdparty_commit_hash }}
doris_version: ${{ steps.check_diff.outputs.doris_version }}
@@ -62,6 +67,9 @@ jobs:
- name: Check Diff
id: check_diff
+ env:
+ MANUAL_FORCE_BUILD: ${{ github.event_name == 'workflow_dispatch' &&
github.event.inputs.force_build == 'true' }}
+ MANUAL_DOCKER_ONLY: ${{ github.event_name == 'workflow_dispatch' &&
github.event.inputs.docker_only == 'true' }}
run: |
if [[ -z "$(gh release list)" ]] ||
! gh release list | awk -F "\t" '{ print $3 }' | grep
'automation' >/dev/null; then
@@ -88,17 +96,21 @@ jobs:
echo "Thirdparty Commit: ${thirdparty_commit_hash}"
should_release=false
+ should_update_docker=false
attempt=1
if [[ -z "${last_version}" ]]; then
echo "The first release was detected."
should_release=true
+ should_update_docker=true
elif [[ "${last_version}" != "${current_version}" ]]; then
- cmd="git diff --name-only ${last_version} ${current_version} |
grep -E '^thirdparty/'"
- echo "Execute: ${cmd}"
- content="$(eval "${cmd}")" || true
- if [[ -n "${content}" ]]; then
- echo -e "Detect changes:\n${content}"
+ changed_files="$(git diff --name-only "${last_version}"
"${current_version}")"
+ echo -e "Changed files:\n${changed_files}"
+ if grep -qE '^thirdparty/' <<<"${changed_files}"; then
should_release=true
+ should_update_docker=true
+ fi
+ if grep -qx 'docker/compilation/Dockerfile' <<<"${changed_files}";
then
+ should_update_docker=true
fi
elif [[ "${last_status}" == 'FAILURE' ]] && [[ "${last_attempt}" -lt
"${max_attempts}" ]]; then
# Downloads from third party mirrors fail often enough that a
single
@@ -107,6 +119,25 @@ jobs:
attempt=$((last_attempt + 1))
echo "Previous build failed, retrying (attempt
${attempt}/${max_attempts})."
should_release=true
+ should_update_docker=true
+ fi
+
+ # docker_only takes precedence over force_build because force_build
+ # defaults to true for backwards compatibility with existing callers.
+ if [[ "${MANUAL_DOCKER_ONLY}" == 'true' ]]; then
+ echo 'Manual Docker-only build requested.'
+ should_release=false
+ should_update_docker=true
+ elif [[ "${MANUAL_FORCE_BUILD}" == 'true' ]]; then
+ echo 'Manual full build requested.'
+ should_release=true
+ should_update_docker=true
+ fi
+
+ # Docker-only failures are deterministic image/workflow failures. Do
+ # not turn their scheduled retry into an expensive full thirdparty
build.
+ if [[ "${should_release}" == 'false' ]] && [[
"${should_update_docker}" == 'true' ]]; then
+ attempt="${max_attempts}"
fi
# The macOS x86_64 leg is by far the slowest of the four - 4h22m41s
of the
@@ -129,7 +160,7 @@ jobs:
fi
fi
- if "${should_release}"; then
+ if "${should_release}" || "${should_update_docker}"; then
echo -ne "Update Time: *$(date)*\nDoris Version:
*${current_version}*\nStatus: *BUILDING*\nAttempts: *${attempt}*"
>release_note.md
else
gh release view automation | sed -n '/--/,$p' | awk '{ if (NR > 1)
print $0 }' | sed "{
@@ -139,11 +170,14 @@ jobs:
fi
gh release edit -F release_note.md automation
- echo "should_release=${should_release}" >> $GITHUB_OUTPUT
- echo "should_build_macos_x86=${should_build_macos_x86}" >>
$GITHUB_OUTPUT
- echo "thirdparty_commit_hash=${thirdparty_commit_hash}" >>
$GITHUB_OUTPUT
- echo "doris_version=${current_version}" >> $GITHUB_OUTPUT
- echo "attempt=${attempt}" >> $GITHUB_OUTPUT
+ {
+ echo "should_release=${should_release}"
+ echo "should_update_docker=${should_update_docker}"
+ echo "should_build_macos_x86=${should_build_macos_x86}"
+ echo "thirdparty_commit_hash=${thirdparty_commit_hash}"
+ echo "doris_version=${current_version}"
+ echo "attempt=${attempt}"
+ } >> "${GITHUB_OUTPUT}"
- name: Download Source and Upload
if: steps.check_diff.outputs.should_release == 'true'
@@ -166,7 +200,7 @@ jobs:
build-linux-x86_64:
name: Build
needs: prerelease
- if: needs.prerelease.outputs.should_release == 'true' ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.force_build ==
'true')
+ if: needs.prerelease.outputs.should_release == 'true'
permissions:
contents: write
uses: ./.github/workflows/build-target.yml
@@ -178,7 +212,7 @@ jobs:
build-linux-arm64:
name: Build
needs: prerelease
- if: needs.prerelease.outputs.should_release == 'true' ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.force_build ==
'true')
+ if: needs.prerelease.outputs.should_release == 'true'
permissions:
contents: write
uses: ./.github/workflows/build-target.yml
@@ -190,7 +224,7 @@ jobs:
build-macos-arm64:
name: Build
needs: prerelease
- if: needs.prerelease.outputs.should_release == 'true' ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.force_build ==
'true')
+ if: needs.prerelease.outputs.should_release == 'true'
permissions:
contents: write
uses: ./.github/workflows/build-target.yml
@@ -203,7 +237,7 @@ jobs:
build-macos-x86_64:
name: Build
needs: prerelease
- if: needs.prerelease.outputs.should_build_macos_x86 == 'true' ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.force_build ==
'true')
+ if: needs.prerelease.outputs.should_build_macos_x86 == 'true'
permissions:
contents: write
uses: ./.github/workflows/build-target.yml
@@ -220,7 +254,7 @@ jobs:
if: |
always() &&
needs.build-linux-x86_64.result != 'cancelled' &&
- (needs.prerelease.outputs.should_release == 'true' || github.event_name
== 'workflow_dispatch')
+ needs.prerelease.outputs.should_update_docker == 'true'
runs-on: ubuntu-latest
env:
GH_REPO: ${{ github.repository }}
@@ -283,9 +317,8 @@ jobs:
- name: Prepare Dockerfiles
run: |
- # Pull the upstream Dockerfile from apache/doris to stay in sync with
- # any toolchain / dependency changes they make.
- curl -fsSL
https://raw.githubusercontent.com/apache/doris/master/docker/compilation/Dockerfile
\
+ # Bind the image to the exact Doris revision selected by prerelease.
+ curl -fsSL "https://raw.githubusercontent.com/apache/doris/${{
needs.prerelease.outputs.doris_version }}/docker/compilation/Dockerfile" \
-o Dockerfile.upstream
python3 - << 'PYEOF'
@@ -294,22 +327,7 @@ jobs:
with open('Dockerfile.upstream') as f:
content = f.read()
- # Patch 1: fix epel metalink.
- # The upstream RUN line installs epel-release then immediately runs
yum
- # install/clean/makecache, but epel.repo's metalink may be
unreliable.
- # Insert a sed fix between epel-release install and the next yum
install.
- old_epel = 'yum install epel-release -y && yum install
https://packages.endpointdev.com'
- new_epel = ('yum install epel-release -y \\\n'
- ' && sed -i \\\n'
- ' -e \'s/^metalink=/#metalink=/\' \\\n'
- ' -e
\'s|^#baseurl=http://download.fedoraproject.org/pub/epel/7|baseurl=https://mirrors.aliyun.com/epel/7|\'
\\\n'
- ' /etc/yum.repos.d/epel*.repo \\\n'
- ' && yum install https://packages.endpointdev.com')
- assert old_epel in content, f"Patch 1 failed: target string not
found in upstream Dockerfile"
- patched = content.replace(old_epel, new_epel, 1)
- assert patched != content, "Patch 1 was a no-op: upstream Dockerfile
may have changed"
-
- # Patch 2: replace "clone & build thirdparty" block with downloading
+ # Patch 1: replace "clone & build thirdparty" block with downloading
# our prebuilt artifact. The block starts with "# clone lastest
source
# code" comment and ends with "rm -rf ${DEFAULT_DIR}/doris".
prebuilt_block = (
@@ -324,22 +342,22 @@ jobs:
patched_2 = re.sub(
r'# clone lastest source code.*?rm -rf
\$\{DEFAULT_DIR\}/doris\n',
prebuilt_block,
- patched, flags=re.DOTALL
+ content, flags=re.DOTALL
)
- assert patched_2 != patched, "Patch 2 was a no-op: 'clone lastest
source code' block not found in upstream Dockerfile"
+ assert patched_2 != content, "Patch 1 was a no-op: 'clone lastest
source code' block not found in upstream Dockerfile"
patched = patched_2
# Write normal image Dockerfile
with open('Dockerfile.patched', 'w') as f:
f.write(patched)
- # Patch 3 (no-avx2): add USE_AVX2=0 to the ENV block in builder
stage.
+ # Patch 2 (no-avx2): add USE_AVX2=0 to the ENV block in builder
stage.
# The ENV block ends with PATH=... just before "# install ccache".
old_avx2 = 'PATH="/var/local/ldb-toolchain/bin/:$PATH"\n #
USE_AVX2=0'
new_avx2 = 'PATH="/var/local/ldb-toolchain/bin/:$PATH" \\\n
USE_AVX2=0'
- assert old_avx2 in patched, "Patch 3 failed: '# USE_AVX2=0' comment
not found; upstream Dockerfile may have changed"
+ assert old_avx2 in patched, "Patch 2 failed: '# USE_AVX2=0' comment
not found; upstream Dockerfile may have changed"
noavx2 = patched.replace(old_avx2, new_avx2)
- assert noavx2 != patched, "Patch 3 was a no-op: no-avx2 Dockerfile
is identical to normal Dockerfile"
+ assert noavx2 != patched, "Patch 2 was a no-op: no-avx2 Dockerfile
is identical to normal Dockerfile"
with open('Dockerfile.patched-noavx2', 'w') as f:
f.write(noavx2)
@@ -383,7 +401,7 @@ jobs:
# leave the release note stuck at BUILDING forever.
if: |
!cancelled() && !failure() &&
- (needs.prerelease.outputs.should_release == 'true' || (github.event_name
== 'workflow_dispatch' && github.event.inputs.force_build == 'true'))
+ (needs.prerelease.outputs.should_release == 'true' ||
needs.prerelease.outputs.should_update_docker == 'true')
runs-on: ubuntu-latest
env:
GH_REPO: ${{ github.repository }}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
