This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ratis.git
The following commit(s) were added to refs/heads/master by this push:
new 6fd8ae129 RATIS-2210. Reduce duplication in CI workflow (#1195)
6fd8ae129 is described below
commit 6fd8ae1293044973c2cd22ba4800cea35ebf9f34
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue Dec 24 19:22:44 2024 +0100
RATIS-2210. Reduce duplication in CI workflow (#1195)
---
.github/workflows/check.yml | 203 +++++++++++++++++++++++++++++
.github/workflows/ci.yml | 141 ++++++++++++++++++++
.github/workflows/post-commit.yml | 261 ++------------------------------------
dev-support/checks/compile.sh | 31 +++++
4 files changed, 385 insertions(+), 251 deletions(-)
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
new file mode 100644
index 000000000..b4498927a
--- /dev/null
+++ b/.github/workflows/check.yml
@@ -0,0 +1,203 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This reusable workflow executes a single check from `dev-support/checks/`.
+# Before and after the check, it performs various steps based on workflow
inputs.
+
+name: ci-check
+
+on:
+ workflow_call:
+ inputs:
+ # REQUIRED
+ script:
+ type: string
+ description: "Test script to run from dev-support/checks, without .sh
extension"
+ required: true
+
+ # OPTIONAL (ordered alphabetically)
+ java-version:
+ type: string
+ description: "Java version to set up (default: 8)"
+ default: '8'
+ required: false
+
+ needs-binary-tarball:
+ type: boolean
+ description: "Whether to download Ratis binary tarball created by
build (default: no)"
+ default: false
+ required: false
+
+ needs-maven-repo:
+ type: boolean
+ description: "Whether to download Ratis jars created by build
(default: no)"
+ default: false
+ required: false
+
+ needs-source-tarball:
+ type: boolean
+ description: "Whether to download Ratis source tarball created by
build (default: no)"
+ default: false
+ required: false
+
+ runner:
+ type: string
+ description: "GitHub Actions runner to use"
+ default: 'ubuntu-20.04'
+ required: false
+
+ script-args:
+ type: string
+ description: "Arguments for the test script"
+ default: ''
+ required: false
+
+ split:
+ type: string
+ description: "Name of split for matrix jobs, only used in display name"
+ default: ''
+ required: false
+
+ timeout-minutes:
+ type: number
+ description: "Job timeout in minutes (default: 30)"
+ default: 30
+ required: false
+
+env:
+ MAVEN_ARGS: --batch-mode --show-version
+ MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3
+ WITH_COVERAGE: true
+
+jobs:
+ check:
+ name: ${{ (inputs.split && format('{0} ({1})', inputs.script,
inputs.split)) || inputs.script }}
+ runs-on: ${{ inputs.runner }}
+ timeout-minutes: ${{ inputs.timeout-minutes }}
+ steps:
+ - name: Checkout project
+ if: ${{ !inputs.needs-source-tarball }}
+ uses: actions/checkout@v4
+
+ - name: Download source tarball
+ if: ${{ inputs.needs-source-tarball }}
+ uses: actions/download-artifact@v4
+ with:
+ name: ratis-src
+
+ - name: Extract source tarball
+ if: ${{ inputs.needs-source-tarball }}
+ run: |
+ tar --strip-components 1 -xzvf ratis*-src.tar.gz
+
+ - name: Create cache for Maven dependencies
+ if: ${{ inputs.script == 'build' }}
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/ratis
+ key: maven-repo-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ maven-repo-
+
+ - name: Restore cache for Maven dependencies
+ if: ${{ inputs.script != 'build' }}
+ uses: actions/cache/restore@v4
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/ratis
+ key: maven-repo-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ maven-repo-
+
+ - name: Download Maven repo
+ id: download-maven-repo
+ if: ${{ inputs.needs-maven-repo }}
+ uses: actions/download-artifact@v4
+ with:
+ name: maven-repo
+ path: |
+ ~/.m2/repository/org/apache/ratis
+
+ - name: Download binary tarball
+ if: ${{ inputs.needs-binary-tarball }}
+ uses: actions/download-artifact@v4
+ with:
+ name: ratis-bin
+
+ - name: Extract binary tarball
+ if: ${{ inputs.needs-binary-tarball }}
+ run: |
+ mkdir -p ratis-assembly/target
+ tar xzvf ratis-*-bin.tar.gz -C ratis-assembly/target
+
+ - name: Setup java ${{ inputs.java-version }}
+ if: ${{ inputs.java-version }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: ${{ inputs.java-version }}
+
+ - name: Execute tests
+ run: |
+ dev-support/checks/${{ inputs.script }}.sh ${{ inputs.script-args }}
+ env:
+ WITH_COVERAGE: ${{ inputs.with-coverage }}
+
+ - name: Summary of failures
+ if: ${{ failure() }}
+ run: |
+ if [[ -s "target/${{ inputs.script }}/summary.txt" ]]; then
+ cat target/${{ inputs.script }}/summary.txt
+ fi
+
+ - name: Archive build results
+ if: ${{ !cancelled() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ (inputs.split && format('{0}-{1}', inputs.script,
inputs.split)) || inputs.script }}
+ path: target/${{ inputs.script }}
+ continue-on-error: true
+
+ # The following steps are hard-coded to be run only for 'build' check,
+ # to avoid the need for 3 more inputs.
+ - name: Store binaries for tests
+ if: ${{ inputs.script == 'build' && !cancelled() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: ratis-bin
+ path: |
+ ratis-assembly/target/ratis-assembly-*-bin.tar.gz
+ retention-days: 1
+
+ - name: Store source tarball for compilation
+ if: ${{ inputs.script == 'build' && !cancelled() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: ratis-src
+ path: |
+ ratis-assembly/target/ratis-assembly-*-src.tar.gz
+ retention-days: 1
+
+ - name: Store Maven repo for tests
+ if: ${{ inputs.script == 'build' && !cancelled() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: maven-repo
+ path: |
+ ~/.m2/repository/org/apache/ratis
+ retention-days: 1
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 000000000..11fb1bd51
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,141 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: CI
+
+on:
+ workflow_call:
+ inputs:
+ ref:
+ type: string
+ description: Ratis git ref (branch, tag or commit hash)
+ default: ''
+ required: false
+
+jobs:
+ build:
+ uses: ./.github/workflows/check.yml
+ with:
+ script: build
+ script-args: -Prelease
+ timeout-minutes: 30
+ secrets: inherit
+
+ compile:
+ needs:
+ - build
+ strategy:
+ matrix:
+ java: [ 11, 17, 21 ]
+ fail-fast: false
+ uses: ./.github/workflows/check.yml
+ with:
+ java-version: ${{ matrix.java }}
+ needs-source-tarball: true
+ script: compile
+ script-args: -Djavac.version=${{ matrix.java }}
+ split: ${{ matrix.java }}
+ timeout-minutes: 30
+ secrets: inherit
+
+ repro:
+ needs:
+ - build
+ uses: ./.github/workflows/check.yml
+ with:
+ needs-maven-repo: true
+ script: repro
+ script-args: -Prelease
+ timeout-minutes: 30
+ secrets: inherit
+
+ basic:
+ strategy:
+ matrix:
+ check:
+ - author
+ - checkstyle
+ - findbugs
+ - rat
+ fail-fast: false
+ uses: ./.github/workflows/check.yml
+ with:
+ script: ${{ matrix.check }}
+ timeout-minutes: 30
+ secrets: inherit
+
+ unit:
+ strategy:
+ matrix:
+ profile:
+ - grpc
+ - server
+ - misc
+ fail-fast: false
+ uses: ./.github/workflows/check.yml
+ with:
+ script: unit
+ script-args: -P${{ matrix.profile }}-tests
+ split: ${{ matrix.profile }}
+ timeout-minutes: 60
+ secrets: inherit
+
+ coverage:
+ needs:
+ - build
+ - unit
+ runs-on: ubuntu-20.04
+ timeout-minutes: 30
+ if: (github.repository == 'apache/ratis' || github.repository ==
'apache/incubator-ratis') && github.event_name != 'pull_request'
+ steps:
+ - name: Checkout project
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ - name: Cache for maven dependencies
+ uses: actions/cache/restore@v4
+ with:
+ path: |
+ ~/.m2/repository
+ !~/.m2/repository/org/apache/ratis
+ key: maven-repo-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ maven-repo-
+ - name: Setup java 17
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: 17
+ - name: Download artifacts
+ uses: actions/download-artifact@v4
+ with:
+ path: target/artifacts
+ - name: Untar binaries
+ run: |
+ mkdir -p ratis-assembly/target
+ tar xzvf target/artifacts/ratis-bin/ratis-assembly-*.tar.gz -C
ratis-assembly/target
+ - name: Calculate combined coverage
+ run: ./dev-support/checks/coverage.sh
+ - name: Upload coverage to Sonar
+ run: ./dev-support/checks/sonar.sh
+ env:
+ SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ - name: Archive build results
+ uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: ${{ github.job }}
+ path: target/${{ github.job }}
diff --git a/.github/workflows/post-commit.yml
b/.github/workflows/post-commit.yml
index 5fee9462a..9b0df4ce4 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -12,7 +12,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
name: build-branch
+
on:
push:
branches-ignore:
@@ -20,255 +22,12 @@ on:
tags:
- '**'
pull_request:
-env:
- WITH_COVERAGE: true
+
+concurrency:
+ group: ci-${{ github.event.pull_request.number || github.sha }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
jobs:
- build:
- runs-on: ubuntu-20.04
- timeout-minutes: 30
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- - name: Cache for maven dependencies
- uses: actions/cache@v4
- with:
- path: |
- ~/.m2/repository
- !~/.m2/repository/org/apache/ratis
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Setup java
- uses: actions/setup-java@v4
- with:
- distribution: 'temurin'
- java-version: 8
- - name: Run a full build
- run: ./dev-support/checks/build.sh -Prelease
- - name: Store binaries for tests
- uses: actions/upload-artifact@v4
- with:
- name: ratis-bin
- path: |
- ratis-assembly/target/ratis-assembly-*.tar.gz
- !ratis-assembly/target/ratis-assembly-*-src.tar.gz
- retention-days: 1
- - name: Store source tarball for compilation
- uses: actions/upload-artifact@v4
- with:
- name: ratis-src
- path: ratis-assembly/target/ratis-assembly-*-src.tar.gz
- retention-days: 1
- compile:
- needs:
- - build
- runs-on: ubuntu-20.04
- timeout-minutes: 30
- strategy:
- matrix:
- java: [ 11, 17, 21 ]
- fail-fast: false
- steps:
- - name: Download source tarball
- uses: actions/download-artifact@v4
- with:
- name: ratis-src
- - name: Untar sources
- run: |
- tar --strip-components 1 -xzvf ratis-assembly-*-src.tar.gz
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository
- !~/.m2/repository/org/apache/ratis
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Setup java
- uses: actions/setup-java@v4
- with:
- distribution: 'temurin'
- java-version: ${{ matrix.java }}
- - name: Run a full build
- run: ./dev-support/checks/build.sh -Djavac.version=${{ matrix.java }}
- - name: Test reproducibility
- run: ./dev-support/checks/repro.sh -Djavac.version=${{ matrix.java }}
- rat:
- name: rat
- runs-on: ubuntu-20.04
- timeout-minutes: 15
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository
- !~/.m2/repository/org/apache/ratis
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Run tests
- run: ./dev-support/checks/rat.sh
- - name: Upload results
- uses: actions/upload-artifact@v4
- if: always()
- with:
- name: rat
- path: target/rat
- author:
- name: author
- runs-on: ubuntu-20.04
- timeout-minutes: 15
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- - name: Run tests
- run: ./dev-support/checks/author.sh
- - name: Upload results
- uses: actions/upload-artifact@v4
- if: always()
- with:
- name: author
- path: target/author
- unit:
- name: unit
- runs-on: ubuntu-20.04
- timeout-minutes: 60
- strategy:
- matrix:
- profile:
- - grpc
- - server
- - misc
- fail-fast: false
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository
- !~/.m2/repository/org/apache/ratis
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Setup java
- uses: actions/setup-java@v4
- with:
- distribution: 'temurin'
- java-version: 8
- - name: Run tests
- run: ./dev-support/checks/unit.sh -P${{ matrix.profile }}-tests
- - name: Summary of failures
- run: cat target/${{ github.job }}/summary.txt
- if: ${{ !cancelled() }}
- - name: Upload results
- uses: actions/upload-artifact@v4
- if: ${{ !cancelled() }}
- with:
- name: unit-${{ matrix.profile }}
- path: target/unit
- checkstyle:
- name: checkstyle
- runs-on: ubuntu-20.04
- timeout-minutes: 15
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository
- !~/.m2/repository/org/apache/ratis
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Run tests
- run: ./dev-support/checks/checkstyle.sh
- - name: Upload results
- uses: actions/upload-artifact@v4
- if: always()
- with:
- name: checkstyle
- path: target/checkstyle
- findbugs:
- name: findbugs
- runs-on: ubuntu-20.04
- timeout-minutes: 30
- steps:
- - name: Setup java
- uses: actions/setup-java@v4
- with:
- distribution: 'temurin'
- java-version: 8
- - name: Checkout project
- uses: actions/checkout@v4
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository
- !~/.m2/repository/org/apache/ratis
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Run tests
- run: ./dev-support/checks/findbugs.sh
- - name: Upload results
- uses: actions/upload-artifact@v4
- if: always()
- with:
- name: findbugs
- path: target/findbugs
- coverage:
- needs:
- - build
- - unit
- runs-on: ubuntu-20.04
- timeout-minutes: 30
- if: (github.repository == 'apache/ratis' || github.repository ==
'apache/incubator-ratis') && github.event_name != 'pull_request'
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- with:
- fetch-depth: 0
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository
- !~/.m2/repository/org/apache/ratis
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Setup java 17
- uses: actions/setup-java@v4
- with:
- distribution: 'temurin'
- java-version: 17
- - name: Download artifacts
- uses: actions/download-artifact@v4
- with:
- path: target/artifacts
- - name: Untar binaries
- run: |
- mkdir -p ratis-assembly/target
- tar xzvf target/artifacts/ratis-bin/ratis-assembly-*.tar.gz -C
ratis-assembly/target
- - name: Calculate combined coverage
- run: ./dev-support/checks/coverage.sh
- - name: Upload coverage to Sonar
- run: ./dev-support/checks/sonar.sh
- env:
- SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- - name: Archive build results
- uses: actions/upload-artifact@v4
- if: always()
- with:
- name: ${{ github.job }}
- path: target/${{ github.job }}
+ CI:
+ uses: ./.github/workflows/ci.yml
+ secrets: inherit
diff --git a/dev-support/checks/compile.sh b/dev-support/checks/compile.sh
new file mode 100755
index 000000000..fba4394de
--- /dev/null
+++ b/dev-support/checks/compile.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../.." || exit 1
+
+source "${DIR}/../find_maven.sh"
+
+: ${WITH_COVERAGE:="false"}
+
+MAVEN_OPTIONS='-V -B -Dmaven.javadoc.skip=true -DskipTests'
+
+if [[ "${WITH_COVERAGE}" != "true" ]]; then
+ MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip"
+fi
+
+export MAVEN_OPTS="-Xmx4096m"
+${MVN} ${MAVEN_OPTIONS} clean verify "$@"
+exit $?
