This is an automated email from the ASF dual-hosted git repository.
davidarthur pushed a commit to branch 4.0
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/4.0 by this push:
new e904556ec9e Backport some GHA changes from trunk (#18428)
e904556ec9e is described below
commit e904556ec9e6aee2c36eb53710ce6492c8236412
Author: David Arthur <[email protected]>
AuthorDate: Wed Jan 8 09:57:33 2025 -0500
Backport some GHA changes from trunk (#18428)
Reviewers: Chia-Ping Tsai <[email protected]>
---
.github/actions/run-gradle/action.yml | 4 ++--
.github/scripts/junit.py | 19 +++++++++++++-----
.github/workflows/build.yml | 37 ++++++++++++++++++++++-------------
.github/workflows/ci.yml | 3 +--
4 files changed, 40 insertions(+), 23 deletions(-)
diff --git a/.github/actions/run-gradle/action.yml
b/.github/actions/run-gradle/action.yml
index a5bc4e55237..18d6bdeb1f5 100644
--- a/.github/actions/run-gradle/action.yml
+++ b/.github/actions/run-gradle/action.yml
@@ -16,8 +16,8 @@
# under the License.
#
---
-name: "Gradle Setup"
-description: "Setup Java and Gradle"
+name: "Run Tests with Gradle"
+description: "Run specified Gradle test tasks with configuration for timeout
and test catalog."
inputs:
# Composite actions do not support typed parameters. Everything is treated
as a string
# See: https://github.com/actions/runner/issues/2238
diff --git a/.github/scripts/junit.py b/.github/scripts/junit.py
index 142e6c09076..b23f444720b 100644
--- a/.github/scripts/junit.py
+++ b/.github/scripts/junit.py
@@ -402,10 +402,19 @@ if __name__ == "__main__":
print("<hr/>")
# Print special message if there was a timeout
- exit_code = get_env("GRADLE_EXIT_CODE", int)
- if exit_code == 124:
+ test_exit_code = get_env("GRADLE_TEST_EXIT_CODE", int)
+ quarantined_test_exit_code = get_env("GRADLE_QUARANTINED_TEST_EXIT_CODE",
int)
+
+ if test_exit_code == 124 or quarantined_test_exit_code == 124:
+ # Special handling for timeouts. The exit code 124 is emitted by
'timeout' command used in build.yml.
+ # A watchdog script "thread-dump.sh" will use jstack to force a thread
dump for any Gradle process
+ # still running after the timeout. We capture the exit codes of the
two test tasks and pass them to
+ # this script. If either "test" or "quarantinedTest" fails due to
timeout, we want to fail the overall build.
thread_dump_url = get_env("THREAD_DUMP_URL")
- logger.debug(f"Gradle command timed out. These are partial results!")
+ if test_exit_code == 124:
+ logger.debug(f"Gradle task for 'test' timed out. These are partial
results!")
+ else:
+ logger.debug(f"Gradle task for 'quarantinedTest' timed out. These
are partial results!")
logger.debug(summary)
if thread_dump_url:
print(f"\nThe JUnit tests were cancelled due to a timeout. Thread
dumps were generated before the job was cancelled. "
@@ -414,7 +423,7 @@ if __name__ == "__main__":
else:
logger.debug(f"Failing this step because the tests timed out.
Thread dumps were not archived, check logs in JUnit step.")
exit(1)
- elif exit_code in (0, 1):
+ elif test_exit_code in (0, 1):
logger.debug(summary)
if total_failures > 0:
logger.debug(f"Failing this step due to {total_failures} test
failures")
@@ -425,5 +434,5 @@ if __name__ == "__main__":
else:
exit(0)
else:
- logger.debug(f"Gradle had unexpected exit code {exit_code}. Failing
this step")
+ logger.debug(f"Gradle had unexpected exit code {test_exit_code}.
Failing this step")
exit(1)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e860073ecdd..d878a52ddac 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -21,14 +21,10 @@ name: Check and Test
on:
workflow_call:
inputs:
- gradle-cache-read-only:
- description: "Should the Gradle cache be read-only?"
+ is-trunk:
+ description: "Is this a trunk build?"
default: true
type: boolean
- gradle-cache-write-only:
- description: "Should the Gradle cache be write-only?"
- default: false
- type: boolean
is-public-fork:
description: "Is this CI run from a public fork?"
default: true
@@ -105,8 +101,8 @@ jobs:
uses: ./.github/actions/setup-gradle
with:
java-version: 23
- gradle-cache-read-only: ${{ inputs.gradle-cache-read-only }}
- gradle-cache-write-only: ${{ inputs.gradle-cache-write-only }}
+ gradle-cache-read-only: ${{ !inputs.is-trunk }}
+ gradle-cache-write-only: ${{ inputs.is-trunk }}
develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
- name: Compile and validate
env:
@@ -117,7 +113,7 @@ jobs:
# --scan: Publish the build scan. This will only work on PRs
from apache/kafka and trunk
# --no-scan: For public fork PRs, we won't attempt to publish the
scan
run: |
- ./gradlew --build-cache --info $SCAN_ARG check -x test
+ ./gradlew --build-cache --info $SCAN_ARG check siteDocTar -x test
- name: Archive check reports
if: always()
uses: actions/upload-artifact@v4
@@ -137,6 +133,16 @@ jobs:
run: python .github/scripts/rat.py
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
+ - name: Check generated documentation
+ # Check if there are any empty files under ./site-docs/generated, If
any empty files are found, print an error
+ # message and list the empty files
+ run: |
+ tar zxvf core/build/distributions/kafka_2.13-$(./gradlew properties
| grep version: | awk '{print $NF}' | head -n 1)-site-docs.tgz
+ if find ./site-docs/generated -type f -exec grep -L "." {} \; | grep
-q "."; then
+ echo "One or more documentation files are empty!" >&2
+ find ./site-docs/generated -type f -exec grep -L "." {} \; >&2
+ exit 1
+ fi
test:
needs: [validate, load-catalog]
@@ -146,6 +152,8 @@ jobs:
fail-fast: false
matrix:
java: [ 23, 17 ] # If we change these, make sure to adjust
ci-complete.yml
+ outputs:
+ timed-out: ${{ (steps.junit-test.outputs.gradle-exitcode == '124' ||
steps.junit-quarantined-test.outputs.gradle-exitcode == '124') }}
name: JUnit tests Java ${{ matrix.java }}
steps:
- name: Checkout code
@@ -158,8 +166,8 @@ jobs:
uses: ./.github/actions/setup-gradle
with:
java-version: ${{ matrix.java }}
- gradle-cache-read-only: ${{ inputs.gradle-cache-read-only }}
- gradle-cache-write-only: ${{ inputs.gradle-cache-write-only }}
+ gradle-cache-read-only: ${{ !inputs.is-trunk }}
+ gradle-cache-write-only: ${{ inputs.is-trunk }}
develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
# If the load-catalog job failed, we won't be able to download the
artifact. Since we don't want this to fail
@@ -176,7 +184,7 @@ jobs:
uses: ./.github/actions/run-gradle
with:
test-task: quarantinedTest
- timeout-minutes: 30
+ timeout-minutes: 180
test-catalog-path: ${{ steps.load-test-catalog.outputs.download-path
}}/combined-test-catalog.txt
build-scan-artifact-name: build-scan-quarantined-test-${{
matrix.java }}
@@ -225,7 +233,8 @@ jobs:
GITHUB_WORKSPACE: ${{ github.workspace }}
JUNIT_REPORT_URL: ${{
steps.junit-upload-artifact.outputs.artifact-url }}
THREAD_DUMP_URL: ${{
steps.thread-dump-upload-artifact.outputs.artifact-url }}
- GRADLE_EXIT_CODE: ${{ steps.junit-test.outputs.gradle-exitcode }}
+ GRADLE_TEST_EXIT_CODE: ${{ steps.junit-test.outputs.gradle-exitcode
}}
+ GRADLE_QUARANTINED_TEST_EXIT_CODE: ${{
steps.junit-quarantined-test.outputs.gradle-exitcode }}
- name: Archive Test Catalog
if: ${{ always() && matrix.java == '23' }}
@@ -239,7 +248,7 @@ jobs:
update-test-catalog:
name: Update Test Catalog
needs: test
- if: ${{ always() && !inputs.is-public-fork }}
+ if: ${{ always() && inputs.is-trunk && needs.test.outputs.timed-out ==
'false' }}
runs-on: ubuntu-latest
permissions:
contents: write
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a0d531f267d..c02c47c7584 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -38,8 +38,7 @@ jobs:
build:
uses: ./.github/workflows/build.yml
with:
- gradle-cache-read-only: ${{ github.ref != 'refs/heads/trunk' }}
- gradle-cache-write-only: ${{ github.ref == 'refs/heads/trunk' }}
+ is-trunk: ${{ github.ref == 'refs/heads/trunk' }}
is-public-fork: ${{ github.event.pull_request.head.repo.fork || false }}
secrets:
inherit
