This is an automated email from the ASF dual-hosted git repository. adoroszlai pushed a commit to branch branch-1 in repository https://gitbox.apache.org/repos/asf/ratis-thirdparty.git
commit 4cad915eb3e1ab661f0dbb59b9a683fe951ebf3e Author: Doroszlai, Attila <[email protected]> AuthorDate: Thu Jul 2 14:28:15 2026 +0200 RATIS-2579. Split CI build (#134) (cherry picked from commit 799471243f5df2623b9eeb79b12438e7626521de) --- .github/workflows/ci.yaml | 40 ++------- .github/workflows/reusable-check.yaml | 159 ++++++++++++++++++++++++++++++++++ .github/workflows/reusable-ci.yaml | 59 +++++++++++++ dev-support/checks/_diffoscope.sh | 50 +++++++++++ dev-support/checks/_mvn_check.sh | 36 ++++++++ dev-support/checks/build.sh | 24 +++++ dev-support/checks/repro.sh | 24 +++++ dev-support/checks/unit.sh | 24 +++++ 8 files changed, 382 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 46a57df..77dffae 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This workflow defines events and conditions to execute CI. + name: ci on: @@ -30,40 +32,10 @@ concurrency: permissions: { } jobs: - build: - runs-on: ubuntu-24.04 + CI: if: github.event_name == 'pull_request' || github.repository == 'apache/ratis-thirdparty' || github.ref_name != 'master' - steps: - - name: Checkout project - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - name: Restore Maven repo - uses: apache/infrastructure-actions/stash/restore@2245ffcb262ea1723462729b032d1d5c71290dfc - with: - path: ~/.m2/repository - key: maven-repo - - name: Setup Java - uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 - with: - distribution: 'temurin' - java-version: 8 - - name: Run a full build - run: ./mvnw --no-transfer-progress -Ptest -Prelease clean install - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - - name: Check build reproducibility - run: ./mvnw --no-transfer-progress -Ptest -Prelease -DskipTests clean verify artifact:compare - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - - name: Delete temporary build artifacts - run: rm -rf ~/.m2/repository/org/apache/ratis - if: always() - - name: Save Maven repo - uses: apache/infrastructure-actions/stash/save@2245ffcb262ea1723462729b032d1d5c71290dfc - with: - path: ~/.m2/repository - key: maven-repo - retention-days: 90 + uses: ./.github/workflows/reusable-ci.yaml + secrets: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} diff --git a/.github/workflows/reusable-check.yaml b/.github/workflows/reusable-check.yaml new file mode 100644 index 0000000..e7e7d0e --- /dev/null +++ b/.github/workflows/reusable-check.yaml @@ -0,0 +1,159 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This reusable workflow executes a single check from `dev-support/checks/`. +# Before and after the check, it performs various steps based on workflow inputs. + +name: reusable-check + +on: + workflow_call: + inputs: + # REQUIRED + script: + type: string + description: "Test script to run from dev-support/checks, without .sh extension" + required: true + + # OPTIONAL (ordered alphabetically) + java-version: + type: string + description: "Java version to set up (default: 17)" + default: '17' + required: false + + needs-maven-repo: + type: boolean + description: "Whether to download jars created by build (default: no)" + default: false + required: false + + post-failure: + type: string + description: "Command to execute after the test script, if it failed (default: none)" + default: '' + required: false + + runner: + type: string + description: "GitHub Actions runner to use" + default: 'ubuntu-24.04' + required: false + + script-args: + type: string + description: "Arguments for the test script" + default: '' + required: false + + split: + type: string + description: "Name of split for matrix jobs, only used in display name" + default: '' + required: false + + timeout-minutes: + type: number + description: "Job timeout in minutes (default: 30)" + default: 30 + required: false + + secrets: + DEVELOCITY_ACCESS_KEY: + description: 'Token for submitting build scan to Develocity' + required: false + +env: + MAVEN_ARGS: --batch-mode --show-version + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + SCRIPT: ${{ inputs.script }} + +jobs: + check: + name: ${{ (inputs.split && format('{0} ({1})', inputs.script, inputs.split)) || inputs.script }} + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout-minutes }} + steps: + - name: Checkout project + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Restore Maven repo + uses: apache/infrastructure-actions/stash/restore@2245ffcb262ea1723462729b032d1d5c71290dfc + with: + path: ~/.m2/repository + key: maven-repo + + - name: Download Maven repo + id: download-maven-repo + if: ${{ inputs.needs-maven-repo }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ratis-repo + path: | + ~/.m2/repository/org/apache/ratis + + - name: Setup java ${{ inputs.java-version }} + if: ${{ inputs.java-version }} + uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 + with: + distribution: 'temurin' + java-version: ${{ inputs.java-version }} + + - name: Execute tests + run: | + $COMMAND + env: + COMMAND: dev-support/checks/${{ inputs.script }}.sh ${{ inputs.script-args }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + + - name: Execute post-failure steps + if: ${{ failure() && inputs.post-failure }} + run: | + $COMMAND + env: + COMMAND: ${{ inputs.post-failure }} + + - name: Archive build results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ (inputs.split && format('{0}-{1}', inputs.script, inputs.split)) || inputs.script }} + path: target/${{ inputs.script }} + continue-on-error: true + + # The following steps are hard-coded to be run only for 'build' check, + # to avoid the need for more inputs. + - name: Store Maven repo for tests + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ratis-repo + path: | + ~/.m2/repository/org/apache/ratis + retention-days: 1 + + - name: Delete temporary build artifacts + if: always() + run: rm -rf ~/.m2/repository/org/apache/ratis + + - name: Save Maven repo + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: apache/infrastructure-actions/stash/save@2245ffcb262ea1723462729b032d1d5c71290dfc + with: + path: ~/.m2/repository + key: maven-repo + retention-days: 90 diff --git a/.github/workflows/reusable-ci.yaml b/.github/workflows/reusable-ci.yaml new file mode 100644 index 0000000..130bdc5 --- /dev/null +++ b/.github/workflows/reusable-ci.yaml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This reusable workflow defines CI. + +name: reusable-ci + +on: + workflow_call: + secrets: + DEVELOCITY_ACCESS_KEY: + description: 'Token for submitting build scan to Develocity' + required: false + +permissions: { } + +jobs: + build: + uses: ./.github/workflows/reusable-check.yaml + with: + script: build + script-args: -Prelease + secrets: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + + test: + needs: + - build + uses: ./.github/workflows/reusable-check.yaml + with: + needs-maven-repo: true + script: unit + script-args: -pl test + secrets: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + + repro: + needs: + - build + uses: ./.github/workflows/reusable-check.yaml + with: + needs-maven-repo: true + script: repro + script-args: -Prelease + post-failure: dev-support/checks/_diffoscope.sh + secrets: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} diff --git a/dev-support/checks/_diffoscope.sh b/dev-support/checks/_diffoscope.sh new file mode 100755 index 0000000..a6ff461 --- /dev/null +++ b/dev-support/checks/_diffoscope.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Helper script to compare jars reported by maven-artifact-plugin + +set -e -u -o pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../.." || exit 1 + +BASE_DIR="$(pwd -P)" +: ${OUTPUT_LOG:="${BASE_DIR}/target/repro/output.log"} + +if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + sudo apt update -q + sudo apt install -y diffoscope +fi + +for jar in $(grep -o "investigate with diffoscope [^ ]*\.jar [^ ]*\.jar" "${OUTPUT_LOG}" | awk '{ print $NF }'); do + jarname=$(basename "$jar") + if [[ ! -e "$jar" ]]; then + echo "$jar does not exist" + continue + fi + + ref=$(find target/reference -name "$jarname") + if [[ -z "$ref" ]]; then + ref=$(find ~/.m2/repository -name "$jarname") + fi + + if [[ ! -e "$ref" ]]; then + echo "Reference not found for: $jarname" + continue + fi + + diffoscope "$ref" "$jar" +done diff --git a/dev-support/checks/_mvn_check.sh b/dev-support/checks/_mvn_check.sh new file mode 100644 index 0000000..18eda1e --- /dev/null +++ b/dev-support/checks/_mvn_check.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Inputs: +# - CHECK: name of the check, used for output dir +# - arguments: passed to mvn + +set -u -o pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../.." || exit 1 + +source "${DIR}/../find_maven.sh" + +: ${MAVEN_OPTIONS:=""} + +REPORT_DIR="${OUTPUT_DIR:-"${DIR}/../../target/${CHECK}"}" +mkdir -p "$REPORT_DIR" + +export MAVEN_OPTS="-Xmx4096m" +${MVN} --batch-mode --show-version ${MAVEN_OPTIONS} "$@" \ + | tee "${REPORT_DIR}/output.log" +exit $? diff --git a/dev-support/checks/build.sh b/dev-support/checks/build.sh new file mode 100755 index 0000000..f265155 --- /dev/null +++ b/dev-support/checks/build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -u -o pipefail + +CHECK="$( basename "${BASH_SOURCE[0]}" | cut -f1 -d'.' )" +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +MAVEN_OPTIONS='-Dmaven.javadoc.skip=true -DskipTests' + +source "${DIR}/_mvn_check.sh" install "$@" diff --git a/dev-support/checks/repro.sh b/dev-support/checks/repro.sh new file mode 100755 index 0000000..0887a17 --- /dev/null +++ b/dev-support/checks/repro.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -u -o pipefail + +CHECK="$( basename "${BASH_SOURCE[0]}" | cut -f1 -d'.' )" +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +MAVEN_OPTIONS='-Dmaven.javadoc.skip=true -DskipTests' + +source "${DIR}/_mvn_check.sh" verify artifact:compare "$@" diff --git a/dev-support/checks/unit.sh b/dev-support/checks/unit.sh new file mode 100755 index 0000000..cb82dbd --- /dev/null +++ b/dev-support/checks/unit.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -u -o pipefail + +CHECK="$( basename "${BASH_SOURCE[0]}" | cut -f1 -d'.' )" +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +MAVEN_OPTIONS='--fail-at-end -Ptest' + +source "${DIR}/_mvn_check.sh" test "$@"
