This is an automated email from the ASF dual-hosted git repository. ijuma pushed a commit to branch 2.8 in repository https://gitbox.apache.org/repos/asf/kafka.git
commit 0ba00600c674a2079ff29f9c7c71c76bc40a2b63 Author: Ismael Juma <[email protected]> AuthorDate: Mon Apr 5 08:42:04 2021 -0700 KAFKA-12614: Use Jenkinsfile for trunk and release branch builds (#10473) * Run all JDK/Scala version combinations for trunk/release branch builds. * Only retry failures in PR builds for now (we can remove this distinction if/when we report flaky failures as described in KAFKA-12216). * Disable concurrent builds * Send email to dev list on build failure * Use triple double quotes in `doValidation` since we use string interpolation for `SCALA_VERSION`. * Update release.py to output new `Unit/integration tests` Jenkins link Reviewers: Gwen Shapira <[email protected]>, David Arthur <[email protected]> --- Jenkinsfile | 136 +++++++++++++++++++++++++++++++++++++++++++++++++++++------- release.py | 2 +- 2 files changed, 121 insertions(+), 17 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5579887..f9110c6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,19 +26,26 @@ def setupGradle() { } def doValidation() { - sh ''' + sh """ ./gradlew -PscalaVersion=$SCALA_VERSION clean compileJava compileScala compileTestJava compileTestScala \ spotlessScalaCheck checkstyleMain checkstyleTest spotbugsMain rat \ --profile --no-daemon --continue -PxmlSpotBugsReport=true - ''' + """ } -def doTest(target = "unitTest integrationTest") { - sh """ - ./gradlew -PscalaVersion=$SCALA_VERSION ${target} \ - --profile --no-daemon --continue -PtestLoggingEvents=started,passed,skipped,failed \ - -PignoreFailures=true -PmaxParallelForks=2 -PmaxTestRetries=1 -PmaxTestRetryFailures=5 - """ +def isChangeRequest(env) { + env.CHANGE_ID != null && !env.CHANGE_ID.isEmpty() +} + +def retryFlagsString(env) { + if (isChangeRequest(env)) " -PmaxTestRetries=1 -PmaxTestRetryFailures=5" + else "" +} + +def doTest(env, target = "unitTest integrationTest") { + sh """./gradlew -PscalaVersion=$SCALA_VERSION ${target} \ + --profile --no-daemon --continue -PtestLoggingEvents=started,passed,skipped,failed \ + -PignoreFailures=true -PmaxParallelForks=2""" + retryFlagsString(env) junit '**/build/test-results/**/TEST-*.xml' } @@ -95,10 +102,16 @@ def tryStreamsArchetype() { pipeline { agent none + + options { + disableConcurrentBuilds() + } + stages { stage('Build') { parallel { - stage('JDK 8') { + + stage('JDK 8 and Scala 2.12') { agent { label 'ubuntu' } tools { jdk 'jdk_1.8_latest' @@ -114,12 +127,12 @@ pipeline { steps { setupGradle() doValidation() - doTest() + doTest(env) tryStreamsArchetype() } } - stage('JDK 11') { + stage('JDK 11 and Scala 2.13') { agent { label 'ubuntu' } tools { jdk 'jdk_11_latest' @@ -134,12 +147,12 @@ pipeline { steps { setupGradle() doValidation() - doTest() + doTest(env) echo 'Skipping Kafka Streams archetype test for Java 11' } } - - stage('JDK 15') { + + stage('JDK 15 and Scala 2.13') { agent { label 'ubuntu' } tools { jdk 'jdk_15_latest' @@ -154,7 +167,7 @@ pipeline { steps { setupGradle() doValidation() - doTest() + doTest(env) echo 'Skipping Kafka Streams archetype test for Java 15' } } @@ -172,11 +185,102 @@ pipeline { setupGradle() doValidation() catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - doTest('unitTest') + doTest(env, 'unitTest') } echo 'Skipping Kafka Streams archetype test for ARM build' } } + + // To avoid excessive Jenkins resource usage, we only run the stages + // above at the PR stage. The ones below are executed after changes + // are pushed to trunk and/or release branches. We achieve this via + // the `when` clause. + + stage('JDK 8 and Scala 2.13') { + when { + not { changeRequest() } + beforeAgent true + } + agent { label 'ubuntu' } + tools { + jdk 'jdk_1.8_latest' + maven 'maven_3_latest' + } + options { + timeout(time: 8, unit: 'HOURS') + timestamps() + } + environment { + SCALA_VERSION=2.13 + } + steps { + setupGradle() + doValidation() + doTest(env) + tryStreamsArchetype() + } + } + + stage('JDK 11 and Scala 2.12') { + when { + not { changeRequest() } + beforeAgent true + } + agent { label 'ubuntu' } + tools { + jdk 'jdk_11_latest' + } + options { + timeout(time: 8, unit: 'HOURS') + timestamps() + } + environment { + SCALA_VERSION=2.12 + } + steps { + setupGradle() + doValidation() + doTest(env) + echo 'Skipping Kafka Streams archetype test for Java 11' + } + } + + stage('JDK 15 and Scala 2.12') { + when { + not { changeRequest() } + beforeAgent true + } + agent { label 'ubuntu' } + tools { + jdk 'jdk_15_latest' + } + options { + timeout(time: 8, unit: 'HOURS') + timestamps() + } + environment { + SCALA_VERSION=2.12 + } + steps { + setupGradle() + doValidation() + doTest(env) + echo 'Skipping Kafka Streams archetype test for Java 15' + } + } + } + } + } + + post { + always { + script { + if (!isChangeRequest(env)) { + step([$class: 'Mailer', + notifyEveryUnstableBuild: true, + recipients: "[email protected]", + sendToIndividuals: false]) + } } } } diff --git a/release.py b/release.py index 94bb173..ef00ec0 100755 --- a/release.py +++ b/release.py @@ -731,7 +731,7 @@ https://kafka.apache.org/%(docs_version)s/documentation.html https://kafka.apache.org/%(docs_version)s/protocol.html * Successful Jenkins builds for the %(dev_branch)s branch: -Unit/integration tests: https://builds.apache.org/job/kafka-%(dev_branch)s-jdk8/<BUILD NUMBER>/ +Unit/integration tests: https://ci-builds.apache.org/job/Kafka/job/kafka/job/%(dev_branch)s/<BUILD NUMBER>/ System tests: https://jenkins.confluent.io/job/system-test-kafka/job/%(dev_branch)s/<BUILD_NUMBER>/ /**************************************
