This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e385e8fe11 HDDS-6717. Allow running flaky-test-check with specific
Ratis commit (#6675)
e385e8fe11 is described below
commit e385e8fe111470f3c5d28176f950334979016779
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue May 14 10:31:17 2024 +0200
HDDS-6717. Allow running flaky-test-check with specific Ratis commit (#6675)
---
.github/workflows/build-ratis.yml | 137 ++++++++++++++++++++++++++
.github/workflows/intermittent-test-check.yml | 54 +++++++++-
2 files changed, 190 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/build-ratis.yml
b/.github/workflows/build-ratis.yml
new file mode 100644
index 0000000000..a6dd6b08ad
--- /dev/null
+++ b/.github/workflows/build-ratis.yml
@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This workflow can be called by other workflows to build Ratis.
+#
+# Inputs:
+# - Ratis repo
+# - the commit to build
+# Outputs:
+# - various version numbers that need to be provided to the Ozone build
process.
+# - Ratis repository is uploaded as an artifact named `ratis-jars`
+#
+# See `intermittent-test-check.yml` as an example use of this workflow.
+
+name: build-ratis
+on:
+ workflow_call:
+ inputs:
+ repo:
+ description: Ratis repository
+ default: apache/ratis
+ required: true
+ type: string
+ ref:
+ description: Ratis ref (branch, tag or commit SHA)
+ default: master
+ required: true
+ type: string
+ outputs:
+ ratis-version:
+ description: "Ratis Version"
+ value: ${{ jobs.ratis.outputs.ratis-version }}
+ thirdparty-version:
+ description: "Ratis Third-Party Version"
+ value: ${{ jobs.ratis.outputs.thirdparty-version }}
+ grpc-version:
+ description: "gRPC Version"
+ value: ${{ jobs.ratis-thirdparty.outputs.grpc-version }}
+ netty-version:
+ description: "Netty Version"
+ value: ${{ jobs.ratis-thirdparty.outputs.netty-version }}
+ protobuf-version:
+ description: "Protobuf Version"
+ value: ${{ jobs.ratis-thirdparty.outputs.protobuf-version }}
+env:
+ MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3
+jobs:
+ ratis:
+ runs-on: ubuntu-20.04
+ timeout-minutes: 60
+ outputs:
+ ratis-version: ${{ steps.versions.outputs.ratis }}
+ thirdparty-version: ${{ steps.versions.outputs.thirdparty }}
+ steps:
+ - name: Checkout project
+ uses: actions/checkout@v4
+ with:
+ repository: ${{ inputs.repo }}
+ ref: ${{ inputs.ref }}
+ - name: Cache for maven dependencies
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.m2/repository
+ !~/.m2/repository/org/apache/ratis
+ key: ratis-dependencies-${{ hashFiles('**/pom.xml') }}
+ - name: Setup java
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: 8
+ - name: Get component versions
+ id: versions
+ run: |
+ thirdparty_version="$(mvn help:evaluate -N -q -DforceStdout
-Dexpression=ratis.thirdparty.version)"
+ echo "thirdparty=${thirdparty_version}" >> $GITHUB_OUTPUT
+
+ ratis_sha=$(git rev-parse --short HEAD)
+ ratis_version="$(mvn help:evaluate -N -q -DforceStdout
-Dexpression=project.version | sed -e "s/-SNAPSHOT/-${ratis_sha}-SNAPSHOT/")"
+ echo "ratis=${ratis_version}" >> $GITHUB_OUTPUT
+ - name: Run a full build
+ run: |
+ mvn versions:set -DnewVersion=${{ steps.versions.outputs.ratis }}
+ dev-support/checks/build.sh
+ - name: Store Maven repo for tests
+ uses: actions/upload-artifact@v4
+ with:
+ name: ratis-jars
+ path: |
+ ~/.m2/repository/org/apache/ratis
+ retention-days: 1
+ ratis-thirdparty:
+ runs-on: ubuntu-20.04
+ needs:
+ - ratis
+ timeout-minutes: 30
+ outputs:
+ grpc-version: ${{ steps.versions.outputs.grpc }}
+ netty-version: ${{ steps.versions.outputs.netty }}
+ protobuf-version: ${{ steps.versions.outputs.protobuf }}
+ steps:
+ - name: Checkout project
+ uses: actions/checkout@v4
+ with:
+ repository: apache/ratis-thirdparty
+ ref: ${{ needs.ratis.outputs.thirdparty-version }}
+ - name: Get component versions
+ id: versions
+ run: |
+ echo "grpc=$(mvn help:evaluate -N -q -DforceStdout
-Dexpression=shaded.grpc.version)" >> $GITHUB_OUTPUT
+ echo "netty=$(mvn help:evaluate -N -q -DforceStdout
-Dexpression=shaded.netty.version)" >> $GITHUB_OUTPUT
+ echo "protobuf=$(mvn help:evaluate -N -q -DforceStdout
-Dexpression=shaded.protobuf.version)" >> $GITHUB_OUTPUT
+ debug:
+ runs-on: ubuntu-20.04
+ needs:
+ - ratis
+ - ratis-thirdparty
+ steps:
+ - name: Print versions
+ run: |
+ echo ${{ needs.ratis.outputs.ratis-version }}
+ echo ${{ needs.ratis.outputs.thirdparty-version }}
+ echo ${{ needs.ratis-thirdparty.outputs.grpc-version }}
+ echo ${{ needs.ratis-thirdparty.outputs.netty-version }}
+ echo ${{ needs.ratis-thirdparty.outputs.protobuf-version }}
diff --git a/.github/workflows/intermittent-test-check.yml
b/.github/workflows/intermittent-test-check.yml
index dda123305b..100e9ab249 100644
--- a/.github/workflows/intermittent-test-check.yml
+++ b/.github/workflows/intermittent-test-check.yml
@@ -40,12 +40,22 @@ on:
description: Stop after first failure
default: false
required: true
+ ratis-repo:
+ description: Ratis repository
+ default: ''
+ required: false
+ ratis-ref:
+ description: Ratis ref (branch, tag or commit SHA)
+ default: ''
+ required: false
env:
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3
TEST_CLASS: ${{ github.event.inputs.test-class}}
TEST_METHOD: ${{ github.event.inputs.test-name }}
ITERATIONS: ${{ github.event.inputs.iterations }}
FAIL_FAST: ${{ github.event.inputs.fail-fast }}
+ RATIS_REPO: ${{ github.event.inputs.ratis-repo }}
+ RATIS_VERSION: ${{ github.event.inputs.ratis-ref }}
run-name: ${{ github.event_name == 'workflow_dispatch' &&
format('{0}#{1}[{2}]-{3}x{4}', inputs.test-class, inputs.test-name, inputs.ref,
inputs.splits, inputs.iterations) || '' }}
jobs:
prepare-job:
@@ -66,9 +76,17 @@ jobs:
printf -v x "%s," "${splits[@]}"
split_matrix="[${x%,}]"
echo "matrix=$split_matrix" >> $GITHUB_OUTPUT
+ ratis:
+ uses: ./.github/workflows/build-ratis.yml
+ if: ${{ github.event.inputs.ratis-ref != '' }}
+ with:
+ repo: ${{ github.event.inputs.ratis-repo || format('{0}/ratis',
github.repository_owner) }}
+ ref: ${{ github.event.inputs.ratis-ref }}
build:
+ if: ${{ always() }}
needs:
- prepare-job
+ - ratis
runs-on: ubuntu-20.04
timeout-minutes: 60
steps:
@@ -83,13 +101,30 @@ jobs:
key: maven-repo-${{ hashFiles('**/pom.xml') }}
restore-keys: |
maven-repo-
+ - name: Download Ratis repo
+ if: ${{ github.event.inputs.ratis-ref != '' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: ratis-jars
+ path: |
+ ~/.m2/repository/org/apache/ratis
- name: Setup java
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 8
- name: Build (most) of Ozone
- run: hadoop-ozone/dev-support/checks/build.sh -Dskip.npx
-Dskip.installnpx -DskipShade
+ run: |
+ args="-Dskip.npx -Dskip.installnpx -DskipShade"
+ if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then
+ args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version
}}"
+ args="$args -Dratis.thirdparty.version=${{
needs.ratis.outputs.thirdparty-version }}"
+ args="$args -Dio.grpc.version=${{ needs.ratis.outputs.grpc-version
}}"
+ args="$args -Dnetty.version=${{ needs.ratis.outputs.netty-version
}}"
+ args="$args -Dgrpc.protobuf-compile.version=${{
needs.ratis.outputs.protobuf-version }}"
+ fi
+
+ hadoop-ozone/dev-support/checks/build.sh $args
- name: Store Maven repo for tests
uses: actions/upload-artifact@v4
with:
@@ -98,8 +133,10 @@ jobs:
~/.m2/repository/org/apache/ozone
retention-days: 1
run-test:
+ if: ${{ always() }}
needs:
- prepare-job
+ - ratis
- build
name: Run-Split
runs-on: ubuntu-20.04
@@ -120,6 +157,13 @@ jobs:
key: maven-repo-${{ hashFiles('**/pom.xml') }}
restore-keys: |
maven-repo-
+ - name: Download Ratis repo
+ if: ${{ github.event.inputs.ratis-ref != '' }}
+ uses: actions/download-artifact@v4
+ with:
+ name: ratis-jars
+ path: |
+ ~/.m2/repository/org/apache/ratis
- name: Download Ozone repo
id: download-ozone-repo
uses: actions/download-artifact@v4
@@ -140,6 +184,14 @@ jobs:
fi
args="-DexcludedGroups=native|slow|unhealthy"
+ if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then
+ args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version
}}"
+ args="$args -Dratis.thirdparty.version=${{
needs.ratis.outputs.thirdparty-version }}"
+ args="$args -Dio.grpc.version=${{ needs.ratis.outputs.grpc-version
}}"
+ args="$args -Dnetty.version=${{ needs.ratis.outputs.netty-version
}}"
+ args="$args -Dgrpc.protobuf-compile.version=${{
needs.ratis.outputs.protobuf-version }}"
+ fi
+
if [ "$TEST_METHOD" = "ALL" ]; then
echo "Running all tests from $TEST_CLASS"
set -x
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
