This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1bd8d8f56f HDDS-11813. Reduce duplication in CI workflow (#7497)
1bd8d8f56f is described below
commit 1bd8d8f56f13b46ea76e8f18c8804c560e4df1e4
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Mon Mar 10 15:08:01 2025 +0100
HDDS-11813. Reduce duplication in CI workflow (#7497)
---
.github/workflows/check.yml | 289 +++++++++++++
.github/workflows/ci.yml | 592 ++++++--------------------
hadoop-ozone/dev-support/checks/acceptance.sh | 2 +-
3 files changed, 411 insertions(+), 472 deletions(-)
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
new file mode 100644
index 0000000000..6d06484d94
--- /dev/null
+++ b/.github/workflows/check.yml
@@ -0,0 +1,289 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This reusable workflow executes a single check from
`hadoop-ozone/dev-support/checks/`.
+# Before and after the check, it performs various steps based on workflow
inputs.
+
+name: ci-check
+
+on:
+ workflow_call:
+ inputs:
+ # REQUIRED
+ script:
+ type: string
+ description: "Test script to run from hadoop-ozone/dev-support/checks,
without .sh extension"
+ required: true
+
+ sha:
+ type: string
+ description: "Commit SHA to test"
+ required: true
+
+ # OPTIONAL (ordered alphabetically)
+ checkout-fetch-depth:
+ type: number
+ description: "Fetch depth for checking out the repo (default: no
history)"
+ default: 1
+ required: false
+
+ java-version:
+ type: string
+ description: "Java version to set up (default: none)"
+ default: ''
+ required: false
+
+ needs-maven-cache:
+ type: boolean
+ description: "Whether to restore Maven cache before run (default: yes)"
+ default: true
+ required: false
+
+ needs-npm-cache:
+ type: boolean
+ description: "Whether to restore NPM cache before run (default: no)"
+ default: false
+ required: false
+
+ needs-ozone-binary-tarball:
+ type: boolean
+ description: "Whether to download Ozone binary tarball created by
build (default: no)"
+ default: false
+ required: false
+
+ needs-ozone-repo:
+ type: boolean
+ description: "Whether to download Ozone jars created by build
(default: no)"
+ default: false
+ required: false
+
+ needs-ozone-source-tarball:
+ type: boolean
+ description: "Whether to download Ozone source tarball created by
build (default: no)"
+ default: false
+ required: false
+
+ pre-script:
+ type: string
+ description: "Command to execute before the test script (default:
none)"
+ default: ''
+ required: false
+
+ post-failure:
+ type: string
+ description: "Command to execute after the test script, if it failed
(default: none)"
+ default: ''
+ required: false
+
+ post-success:
+ type: string
+ description: "Command to execute after the test script, if it
succeeded (default: none)"
+ default: ''
+ required: false
+
+ ratis-args:
+ type: string
+ description: "Version overrides from custom Ratis build (default:
none)"
+ default: ''
+ required: false
+
+ runner:
+ type: string
+ description: "GitHub Actions runner to use"
+ default: 'ubuntu-24.04'
+ required: false
+
+ script-args:
+ type: string
+ description: "Arguments for the test script, ratis-args are appended"
+ default: ''
+ required: false
+
+ split:
+ type: string
+ description: "Name of split for matrix jobs, only used in display name"
+ default: ''
+ required: false
+
+ timeout-minutes:
+ type: number
+ description: "Job timeout in minutes (default: 30)"
+ default: 30
+ required: false
+
+ with-coverage:
+ type: boolean
+ description: "The value of OZONE_WITH_COVERAGE to set"
+ default: true
+ required: false
+
+env:
+ HADOOP_IMAGE: ghcr.io/apache/hadoop
+ MAVEN_ARGS: --batch-mode --settings ${{ github.workspace
}}/dev-support/ci/maven-settings.xml
+ MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3
+ OZONE_IMAGE: ghcr.io/apache/ozone
+ OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner
+ OZONE_VOLUME_OWNER: 1000
+
+jobs:
+ check:
+ name: ${{ (inputs.split && format('{0} ({1})', inputs.script,
inputs.split)) || inputs.script }}
+ runs-on: ${{ inputs.runner }}
+ timeout-minutes: ${{ inputs.timeout-minutes }}
+ steps:
+ - name: Checkout project
+ if: ${{ !inputs.needs-ozone-source-tarball }}
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ inputs.sha }}
+ fetch-depth: ${{ inputs.checkout-fetch-depth }}
+
+ - name: Download Ozone source tarball
+ if: ${{ inputs.needs-ozone-source-tarball }}
+ uses: actions/download-artifact@v4
+ with:
+ name: ozone-src
+
+ - name: Extract source tarball
+ if: ${{ inputs.needs-ozone-source-tarball }}
+ run: |
+ tar --strip-components 1 -xzvf ozone*-src.tar.gz
+
+ - name: Cache for NPM dependencies
+ if: ${{ inputs.needs-npm-cache }}
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.pnpm-store
+ **/node_modules
+ key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
+ restore-keys: |
+ ${{ runner.os }}-pnpm-
+
+ - name: Cache for Maven dependencies
+ if: ${{ inputs.needs-maven-cache }}
+ uses: actions/cache/restore@v4
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/ozone
+ key: maven-repo-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ maven-repo-
+
+ - name: Download Ozone repo
+ id: download-ozone-repo
+ if: ${{ inputs.needs-ozone-repo }}
+ uses: actions/download-artifact@v4
+ with:
+ name: ozone-repo
+ path: |
+ ~/.m2/repository/org/apache/ozone
+
+ - name: Download Ratis repo
+ if: ${{ inputs.ratis-args != '' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: ratis-jars
+ path: |
+ ~/.m2/repository/org/apache/ratis
+
+ - name: Download Ozone binary tarball
+ if: ${{ inputs.needs-ozone-binary-tarball }}
+ uses: actions/download-artifact@v4
+ with:
+ name: ozone-bin
+
+ - name: Extract binary tarball
+ if: ${{ inputs.needs-ozone-binary-tarball }}
+ run: |
+ mkdir -p hadoop-ozone/dist/target
+ tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
+ rm ozone*.tar.gz
+
+ - name: Setup java ${{ inputs.java-version }}
+ if: ${{ inputs.java-version }}
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: ${{ inputs.java-version }}
+
+ - name: Execute pre-test steps
+ if: ${{ inputs.pre-script }}
+ run: |
+ ${{ inputs.pre-script }}
+
+ - name: Execute tests
+ run: |
+ hadoop-ozone/dev-support/checks/${{ inputs.script }}.sh ${{
inputs.script-args }} ${{ inputs.ratis-args }}
+ env:
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
+ OZONE_WITH_COVERAGE: ${{ inputs.with-coverage }}
+
+ - name: Execute post-failure steps
+ if: ${{ failure() && inputs.post-failure }}
+ run: |
+ ${{ inputs.post-failure }}
+
+ - name: Execute post-success steps
+ if: ${{ !failure() && inputs.post-success }}
+ run: |
+ ${{ inputs.post-success }}
+
+ - name: Summary of failures
+ if: ${{ failure() }}
+ run: |
+ if [[ -s "target/${{ inputs.script }}/summary.md" ]]; then
+ cat target/${{ inputs.script }}/summary.md >> $GITHUB_STEP_SUMMARY
+ fi
+ hadoop-ozone/dev-support/checks/_summary.sh target/${{ inputs.script
}}/summary.txt
+
+ - name: Archive build results
+ if: ${{ !cancelled() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: ${{ (inputs.split && format('{0}-{1}', inputs.script,
inputs.split)) || inputs.script }}
+ path: target/${{ inputs.script }}
+ continue-on-error: true
+
+ # The following steps are hard-coded to be run only for 'build' check,
+ # to avoid the need for 3 more inputs.
+ - name: Store binaries for tests
+ if: ${{ inputs.script == 'build' && !cancelled() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: ozone-bin
+ path: |
+ hadoop-ozone/dist/target/ozone-*.tar.gz
+ !hadoop-ozone/dist/target/ozone-*-src.tar.gz
+ retention-days: 1
+
+ - name: Store source tarball for compilation
+ if: ${{ inputs.script == 'build' && !cancelled() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: ozone-src
+ path: |
+ hadoop-ozone/dist/target/ozone-*-src.tar.gz
+ retention-days: 1
+
+ - name: Store Maven repo for tests
+ if: ${{ inputs.script == 'build' && !cancelled() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: ozone-repo
+ path: |
+ ~/.m2/repository/org/apache/ozone
+ retention-days: 1
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 87193106c2..f68ae51111 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -27,15 +27,14 @@ on:
default: ''
required: false
env:
- FAIL_FAST: ${{ github.event_name == 'pull_request' }}
+ BUILD_ARGS: "-Pdist -Psrc -Dmaven.javadoc.skip=true"
# Minimum required Java version for running Ozone is defined in pom.xml
(javac.version).
TEST_JAVA_VERSION: 21 # JDK version used by CI build and tests; should match
the JDK version in apache/ozone-runner image
+ # MAVEN_ARGS and MAVEN_OPTS are duplicated in check.yml, please keep in sync
MAVEN_ARGS: --batch-mode --settings ${{ github.workspace
}}/dev-support/ci/maven-settings.xml
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3
- HADOOP_IMAGE: ghcr.io/apache/hadoop
- OZONE_IMAGE: ghcr.io/apache/ozone
- OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner
OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }}
+
jobs:
build-info:
runs-on: ubuntu-24.04
@@ -53,6 +52,10 @@ jobs:
needs-integration-tests: ${{
steps.selective-checks.outputs.needs-integration-tests }}
needs-kubernetes-tests: ${{
steps.selective-checks.outputs.needs-kubernetes-tests }}
sha: ${{ steps.get-sha.outputs.sha }}
+ # `env` context cannot be used when calling reusable workflow, so we
need to convert these to `outputs`
+ build-args: ${{ env.BUILD_ARGS }}
+ java-version: ${{ env.TEST_JAVA_VERSION }}
+ with-coverage: ${{ env.OZONE_WITH_COVERAGE }}
steps:
- name: "Checkout ${{ github.ref }} / ${{ github.sha }} (push)"
uses: actions/checkout@v4
@@ -105,80 +108,23 @@ jobs:
env:
ALL_BASIC_CHECKS: "${{ steps.selective-checks.outputs.basic-checks
}}"
run: dev-support/ci/categorize_basic_checks.sh
+
build:
needs:
- build-info
- runs-on: ubuntu-24.04
- timeout-minutes: 60
if: needs.build-info.outputs.needs-build == 'true'
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- with:
- ref: ${{ needs.build-info.outputs.sha }}
- - name: Cache for npm dependencies
- uses: actions/cache@v4
- with:
- path: |
- ~/.pnpm-store
- **/node_modules
- key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
- restore-keys: |
- ${{ runner.os }}-pnpm-
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/ozone
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Download Ratis repo
- if: ${{ inputs.ratis_args != '' }}
- uses: actions/download-artifact@v4
- with:
- name: ratis-jars
- path: |
- ~/.m2/repository/org/apache/ratis
- - name: Setup java ${{ env.TEST_JAVA_VERSION }}
- uses: actions/setup-java@v4
- with:
- distribution: 'temurin'
- java-version: ${{ env.TEST_JAVA_VERSION }}
- - name: Run a full build
- run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc
-Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
- env:
- DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
- - name: Archive build results
- uses: actions/upload-artifact@v4
- if: ${{ !cancelled() }}
- with:
- name: ${{ github.job }}
- path: target/${{ github.job }}
- continue-on-error: true
- - name: Store binaries for tests
- uses: actions/upload-artifact@v4
- with:
- name: ozone-bin
- path: |
- hadoop-ozone/dist/target/ozone-*.tar.gz
- !hadoop-ozone/dist/target/ozone-*-src.tar.gz
- retention-days: 1
- - name: Store source tarball for compilation
- uses: actions/upload-artifact@v4
- if: needs.build-info.outputs.needs-compile == 'true'
- with:
- name: ozone-src
- path: hadoop-ozone/dist/target/ozone-*-src.tar.gz
- retention-days: 1
- - name: Store Maven repo for tests
- uses: actions/upload-artifact@v4
- with:
- name: ozone-repo
- path: |
- ~/.m2/repository/org/apache/ozone
- retention-days: 1
+ uses: ./.github/workflows/check.yml
+ with:
+ java-version: ${{ needs.build-info.outputs.java-version }}
+ needs-npm-cache: true
+ ratis-args: ${{ inputs.ratis_args }}
+ script: build
+ script-args: ${{ needs.build-info.outputs.build-args }}
+ sha: ${{ needs.build-info.outputs.sha }}
+ timeout-minutes: 60
+ with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }}
+ secrets: inherit
+
compile:
needs:
- build-info
@@ -186,7 +132,6 @@ jobs:
- basic
- dependency
- license
- timeout-minutes: 45
if: needs.build-info.outputs.needs-compile == 'true'
strategy:
matrix:
@@ -196,243 +141,80 @@ jobs:
- java: 8
os: macos-13
fail-fast: false
- runs-on: ${{ matrix.os }}
- steps:
- - name: Download Ozone source tarball
- uses: actions/download-artifact@v4
- with:
- name: ozone-src
- - name: Untar sources
- run: |
- tar --strip-components 1 -xzvf ozone*-src.tar.gz
- - name: Workaround for HADOOP-19011
- run: |
- git init
- git config user.name 'Github Actions'
- git config user.email '[email protected]'
- git commit --allow-empty -a -m 'workaround for HADOOP-19011'
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/ozone
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Download Ratis repo
- if: ${{ inputs.ratis_args != '' }}
- uses: actions/download-artifact@v4
- with:
- name: ratis-jars
- path: |
- ~/.m2/repository/org/apache/ratis
- - name: Setup java ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: 'temurin'
- java-version: ${{ matrix.java }}
- - name: Compile Ozone using Java ${{ matrix.java }}
- run: hadoop-ozone/dev-support/checks/compile.sh -Pdist -DskipRecon
-Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{
matrix.java }} ${{ inputs.ratis_args }}
- env:
- OZONE_WITH_COVERAGE: false
- DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
- - name: Archive build results
- uses: actions/upload-artifact@v4
- if: ${{ !cancelled() }}
- with:
- name: ${{ github.job }}-${{ matrix.java }}
- path: target/${{ github.job }}
- continue-on-error: true
+ uses: ./.github/workflows/check.yml
+ with:
+ java-version: ${{ matrix.java }}
+ needs-ozone-source-tarball: true
+ ratis-args: ${{ inputs.ratis_args }}
+ runner: ${{ matrix.os }}
+ script: compile
+ script-args: "-Pdist -DskipRecon -Dmaven.javadoc.failOnWarnings=${{
matrix.java != 8 }} -Djavac.version=${{ matrix.java }}"
+ sha: ${{ needs.build-info.outputs.sha }}
+ split: ${{ matrix.java }}
+ timeout-minutes: 45
+ with-coverage: false
+ secrets: inherit
+
basic:
needs:
- build-info
- runs-on: ubuntu-24.04
- timeout-minutes: 30
if: needs.build-info.outputs.needs-basic-check == 'true'
+ uses: ./.github/workflows/check.yml
+ with:
+ checkout-fetch-depth: ${{ matrix.check != 'bats' && 1 || 0 }}
+ java-version: 8 # HDDS-10150
+ needs-maven-cache: ${{ !contains('author,bats', matrix.check) }}
+ ratis-args: ${{ inputs.ratis_args }}
+ script: ${{ matrix.check }}
+ sha: ${{ needs.build-info.outputs.sha }}
+ timeout-minutes: 30
+ secrets: inherit
strategy:
matrix:
check: ${{ fromJson(needs.build-info.outputs.basic-checks) }}
fail-fast: false
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- with:
- ref: ${{ needs.build-info.outputs.sha }}
- if: matrix.check != 'bats'
- - name: Checkout project with history
- uses: actions/checkout@v4
- with:
- fetch-depth: 0
- ref: ${{ needs.build-info.outputs.sha }}
- if: matrix.check == 'bats'
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/ozone
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- if: ${{ !contains('author,bats', matrix.check) }}
- - name: Download Ratis repo
- if: ${{ inputs.ratis_args != '' }}
- uses: actions/download-artifact@v4
- with:
- name: ratis-jars
- path: |
- ~/.m2/repository/org/apache/ratis
- - name: Setup java 8
- uses: actions/setup-java@v4
- with:
- distribution: 'temurin'
- java-version: 8
- - name: Execute tests
- run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh ${{
inputs.ratis_args }}
- env:
- DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
- - name: Summary of failures
- run: hadoop-ozone/dev-support/checks/_summary.sh target/${{
matrix.check }}/summary.txt
- if: ${{ failure() }}
- - name: Archive build results
- uses: actions/upload-artifact@v4
- if: ${{ !cancelled() }}
- with:
- name: ${{ matrix.check }}
- path: target/${{ matrix.check }}
- continue-on-error: true
+
native:
needs:
- build-info
- basic
- runs-on: ubuntu-24.04
- timeout-minutes: 150
if: needs.build-info.outputs.needs-native-check == 'true'
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- with:
- ref: ${{ needs.build-info.outputs.sha }}
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/ozone
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Download Ratis repo
- if: ${{ inputs.ratis_args != '' }}
- uses: actions/download-artifact@v4
- with:
- name: ratis-jars
- path: |
- ~/.m2/repository/org/apache/ratis
- - name: Setup java ${{ env.TEST_JAVA_VERSION }}
- uses: actions/setup-java@v4
- with:
- distribution: 'temurin'
- java-version: ${{ env.TEST_JAVA_VERSION }}
- - name: Execute tests
- run: hadoop-ozone/dev-support/checks/${{ github.job }}.sh ${{
inputs.ratis_args }}
- env:
- DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
- - name: Summary of failures
- run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job
}}/summary.txt
- if: ${{ failure() }}
- - name: Archive build results
- uses: actions/upload-artifact@v4
- if: ${{ !cancelled() }}
- with:
- name: ${{ github.job }}
- path: target/${{ github.job }}
- continue-on-error: true
+ uses: ./.github/workflows/check.yml
+ secrets: inherit
+ with:
+ java-version: ${{ needs.build-info.outputs.java-version }}
+ ratis-args: ${{ inputs.ratis_args }}
+ script: native
+ sha: ${{ needs.build-info.outputs.sha }}
+ timeout-minutes: 150
+ with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }}
+
dependency:
needs:
- build-info
- build
- runs-on: ubuntu-24.04
- timeout-minutes: 5
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- with:
- ref: ${{ needs.build-info.outputs.sha }}
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/ozone
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Download compiled Ozone binaries
- uses: actions/download-artifact@v4
- with:
- name: ozone-bin
- - name: Untar binaries
- run: |
- mkdir dist
- tar -C dist --strip-components 1 -xzf ozone*.tar.gz
- - name: Execute tests
- run: |
- export OZONE_DIST_DIR=`pwd`/dist
- ./hadoop-ozone/dev-support/checks/dependency.sh
- - name: Archive build results
- uses: actions/upload-artifact@v4
- if: always()
- with:
- name: dependency
- path: target/dependency
- continue-on-error: true
+ uses: ./.github/workflows/check.yml
+ secrets: inherit
+ with:
+ java-version: ${{ needs.build-info.outputs.java-version }}
+ needs-ozone-binary-tarball: true
+ script: dependency
+ sha: ${{ needs.build-info.outputs.sha }}
+ timeout-minutes: 5
+
license:
needs:
- build-info
- build
- runs-on: ubuntu-24.04
- timeout-minutes: 15
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- with:
- ref: ${{ needs.build-info.outputs.sha }}
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/ozone
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Download Ozone repo
- id: download-ozone-repo
- uses: actions/download-artifact@v4
- with:
- name: ozone-repo
- path: |
- ~/.m2/repository/org/apache/ozone
- - name: Setup java ${{ env.TEST_JAVA_VERSION }}
- uses: actions/setup-java@v4
- with:
- distribution: 'temurin'
- java-version: ${{ env.TEST_JAVA_VERSION }}
- - name: Execute tests
- run: |
- hadoop-ozone/dev-support/checks/${{ github.job }}.sh
- - name: Summary of failures
- run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job
}}/summary.txt
- if: ${{ failure() }}
- - name: Archive build results
- uses: actions/upload-artifact@v4
- if: always()
- with:
- name: ${{ github.job }}
- path: target/${{ github.job }}
- continue-on-error: true
+ uses: ./.github/workflows/check.yml
+ secrets: inherit
+ with:
+ java-version: ${{ needs.build-info.outputs.java-version }}
+ needs-ozone-repo: true
+ script: license
+ sha: ${{ needs.build-info.outputs.sha }}
+ timeout-minutes: 15
+
repro:
needs:
- build-info
@@ -440,56 +222,19 @@ jobs:
- basic
- dependency
- license
- runs-on: ubuntu-24.04
- timeout-minutes: 30
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/ozone
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Download Ozone repo
- id: download-ozone-repo
- uses: actions/download-artifact@v4
- with:
- name: ozone-repo
- path: |
- ~/.m2/repository/org/apache/ozone
- - name: Download Ratis repo
- if: ${{ inputs.ratis_args != '' }}
- uses: actions/download-artifact@v4
- with:
- name: ratis-jars
- path: |
- ~/.m2/repository/org/apache/ratis
- - name: Setup java ${{ env.TEST_JAVA_VERSION }}
- uses: actions/setup-java@v4
- with:
- distribution: 'temurin'
- java-version: ${{ env.TEST_JAVA_VERSION }}
- - name: Execute tests
- run: |
- hadoop-ozone/dev-support/checks/${{ github.job }}.sh -Pdist -Psrc
-Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
- - name: Summary of failures
- run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job
}}/summary.txt
- if: ${{ failure() }}
- - name: Check artifact differences
- run: |
- hadoop-ozone/dev-support/checks/_diffoscope.sh
- if: ${{ failure() }}
- - name: Archive build results
- uses: actions/upload-artifact@v4
- if: always()
- with:
- name: ${{ github.job }}
- path: target/${{ github.job }}
- continue-on-error: true
+ uses: ./.github/workflows/check.yml
+ secrets: inherit
+ with:
+ java-version: ${{ needs.build-info.outputs.java-version }}
+ needs-ozone-repo: true
+ ratis-args: ${{ inputs.ratis_args }}
+ script: repro
+ script-args: ${{ needs.build-info.outputs.build-args }}
+ post-failure: hadoop-ozone/dev-support/checks/_diffoscope.sh
+ sha: ${{ needs.build-info.outputs.sha }}
+ timeout-minutes: 30
+ with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }}
+
acceptance:
needs:
- build-info
@@ -497,57 +242,24 @@ jobs:
- basic
- dependency
- license
- runs-on: ubuntu-24.04
- timeout-minutes: 150
if: needs.build-info.outputs.needs-compose-tests == 'true'
+ uses: ./.github/workflows/check.yml
+ secrets: inherit
+ with:
+ java-version: 11 # Hadoop may not work with newer Java
+ needs-ozone-binary-tarball: true
+ ratis-args: ${{ inputs.ratis_args }}
+ script: acceptance
+ script-args: ${{ matrix.suite }}
+ sha: ${{ needs.build-info.outputs.sha }}
+ split: ${{ matrix.suite }}
+ timeout-minutes: 150
+ with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }}
strategy:
matrix:
suite: ${{ fromJson(needs.build-info.outputs.acceptance-suites) }}
fail-fast: false
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- with:
- ref: ${{ needs.build-info.outputs.sha }}
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/ozone
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Download compiled Ozone binaries
- uses: actions/download-artifact@v4
- with:
- name: ozone-bin
- - name: Untar binaries
- run: |
- mkdir -p hadoop-ozone/dist/target
- tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
- rm ozone*.tar.gz
- - name: Setup java 11
- uses: actions/setup-java@v4
- with:
- distribution: 'temurin'
- java-version: 11 # Hadoop may not work with newer Java
- - name: Execute tests
- run: |
- ./hadoop-ozone/dev-support/checks/acceptance.sh
- env:
- KEEP_IMAGE: false
- OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }}
- - name: Summary of failures
- run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job
}}/summary.txt
- if: ${{ failure() }}
- - name: Archive build results
- uses: actions/upload-artifact@v4
- if: always()
- with:
- name: acceptance-${{ matrix.suite }}
- path: target/acceptance
- continue-on-error: true
+
kubernetes:
needs:
- build-info
@@ -555,100 +267,38 @@ jobs:
- basic
- dependency
- license
- runs-on: ubuntu-24.04
- timeout-minutes: 60
if: needs.build-info.outputs.needs-kubernetes-tests == 'true'
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- with:
- ref: ${{ needs.build-info.outputs.sha }}
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/ozone
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Download compiled Ozone binaries
- uses: actions/download-artifact@v4
- with:
- name: ozone-bin
- - name: Untar binaries
- run: |
- mkdir -p hadoop-ozone/dist/target
- tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
- - name: Execute tests
- run: |
- ./hadoop-ozone/dev-support/checks/kubernetes.sh
- - name: Summary of failures
- run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job
}}/summary.txt
- if: ${{ failure() }}
- - name: Archive build results
- uses: actions/upload-artifact@v4
- if: always()
- with:
- name: kubernetes
- path: target/kubernetes
- continue-on-error: true
+ uses: ./.github/workflows/check.yml
+ secrets: inherit
+ with:
+ needs-ozone-binary-tarball: true
+ ratis-args: ${{ inputs.ratis_args }}
+ script: kubernetes
+ sha: ${{ needs.build-info.outputs.sha }}
+ timeout-minutes: 60
+ with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }}
+
integration:
needs:
- build-info
- basic
- runs-on: ubuntu-24.04
- timeout-minutes: 150
if: needs.build-info.outputs.needs-integration-tests == 'true'
+ uses: ./.github/workflows/check.yml
+ secrets: inherit
+ with:
+ java-version: ${{ needs.build-info.outputs.java-version }}
+ ratis-args: ${{ inputs.ratis_args }}
+ script: integration
+ script-args: -Ptest-${{ matrix.profile }}
+ sha: ${{ needs.build-info.outputs.sha }}
+ split: ${{ matrix.profile }}
+ timeout-minutes: 150
+ with-coverage: ${{ fromJSON(needs.build-info.outputs.with-coverage) }}
strategy:
matrix:
profile: ${{ fromJson(needs.build-info.outputs.integration-suites) }}
fail-fast: false
- steps:
- - name: Checkout project
- uses: actions/checkout@v4
- with:
- ref: ${{ needs.build-info.outputs.sha }}
- - name: Cache for maven dependencies
- uses: actions/cache/restore@v4
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/ozone
- key: maven-repo-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- maven-repo-
- - name: Download Ratis repo
- if: ${{ inputs.ratis_args != '' }}
- uses: actions/download-artifact@v4
- with:
- name: ratis-jars
- path: |
- ~/.m2/repository/org/apache/ratis
- - name: Setup java ${{ env.TEST_JAVA_VERSION }}
- uses: actions/setup-java@v4
- with:
- distribution: 'temurin'
- java-version: ${{ env.TEST_JAVA_VERSION }}
- - name: Execute tests
- run: |
- hadoop-ozone/dev-support/checks/integration.sh -Ptest-${{
matrix.profile }} ${{ inputs.ratis_args }}
- env:
- DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
- - name: Summary of failures
- run: |
- if [[ -s "target/${{ github.job }}/summary.md" ]]; then
- cat target/${{ github.job }}/summary.md >> $GITHUB_STEP_SUMMARY
- fi
- hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job
}}/summary.txt
- if: ${{ failure() }}
- - name: Archive build results
- uses: actions/upload-artifact@v4
- if: always()
- with:
- name: it-${{ matrix.profile }}
- path: target/integration
- continue-on-error: true
+
coverage:
runs-on: ubuntu-24.04
timeout-minutes: 30
diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh
b/hadoop-ozone/dev-support/checks/acceptance.sh
index 4a46efe6d6..c286c9b0f2 100755
--- a/hadoop-ozone/dev-support/checks/acceptance.sh
+++ b/hadoop-ozone/dev-support/checks/acceptance.sh
@@ -22,7 +22,7 @@ cd "$DIR/../../.." || exit 1
OZONE_ROOT=$(pwd -P)
: ${HADOOP_AWS_DIR:=""}
-: ${OZONE_ACCEPTANCE_SUITE:=""}
+: ${OZONE_ACCEPTANCE_SUITE:="${1:-}"}
: ${OZONE_TEST_SELECTOR:=""}
: ${OZONE_ACCEPTANCE_TEST_TYPE:="robot"}
: ${OZONE_WITH_COVERAGE:="false"}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
