michaelsembwever commented on code in PR #2852: URL: https://github.com/apache/cassandra/pull/2852#discussion_r1520441105
########## .jenkins/Jenkinsfile: ########## @@ -11,762 +11,480 @@ // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// Se# Licensed to the Apache Software Foundation (ASF) under onee the License for the specific language governing permissions and +// See the License for the specific language governing permissions and // limitations under the License. // // -// Jenkins declaration of how to build and test the current codebase. -// Jenkins infrastructure related settings should be kept in -// https://github.com/apache/cassandra-builds/blob/trunk/jenkins-dsl/cassandra_job_dsl_seed.groovy +// Jenkins CI declaration. +// +// Scripting defs and the declarative pipeline is presented first. +// The stepsMap array describes the pipeline stages to what CI-agnostic scripts they map to. +// +// These CI-agnostic scripts are used as an intermediate dockerized layer above the ant build.xml +// The ant build.xml is never invoked directly. +// +// +// This Jenkinsfile is expected to work on any ci-cassandra.a.o clone. +// Functionality that depends upon ASF Infra and the canonical ci-cassandra.a.o setup (e.g. post-commit builds) +// is required to quietly fail when run on other environments. +// // // Validate/lint this file using the following command // `curl -X POST -F "jenkinsfile=<.jenkins/Jenkinsfile" https://ci-cassandra.apache.org/pipeline-model-converter/validate` +def jdksSupported = ["11", "17"] +def archsSupported = ["amd64", "arm64"] +def pythonsSupported = ["3.8", "3.11"] +def jdkDefault = "11" +def pythonDefault = "3.8" +def stageResults = [:] + +// Steps config +def buildSteps = [ + 'jar': [script: 'build-jars.sh', toCopy: null], + 'artifacts': [script: 'build-artifacts.sh', toCopy: 'apache-cassandra-*.tar.gz,apache-cassandra-*.jar,apache-cassandra-*.pom'], + 'lint': [script: 'check-code.sh', toCopy: null], + 'debian': [script: 'build-debian.sh', toCopy: 'cassandra_*,cassandra-tools_*'], + 'redhat': [script: 'build-redhat.sh rpm', toCopy: '*.rpm'], +] +buildSteps.each() { + it.value.put('type', 'build') + it.value.put('splits', 1) +} + +def testSteps = [ + 'cqlsh-test': [splits: 1], + 'fqltool-test': [splits: 1], + 'test-cdc': [splits: 8], + 'test': [splits: 8], + 'test-trie': [splits: 8], + 'test-compression': [splits: 8], + 'stress-test': [splits: 1], + 'test-burn': [splits: 8], + 'long-test': [splits: 8], + 'test-oa': [splits: 8], + 'test-system-keyspace-directory': [splits: 8], + 'jvm-dtest': [splits: 8], + 'jvm-dtest-upgrade': [splits: 8], + 'simulator-dtest': [splits: 1], + 'dtest': [splits: 64], + 'dtest-novnode': [splits: 64], + 'dtest-offheap': [splits: 64], + 'dtest-large': [splits: 8], + 'dtest-large-novnode': [splits: 8], + 'dtest-upgrade': [splits: 64], + 'dtest-upgrade-novnode': [splits: 64], + 'dtest-upgrade-large': [splits: 64], + 'dtest-upgrade-novnode-large': [splits: 64], +] +testSteps.each() { + it.value.put('type', 'test') + it.value.put('script', '.build/docker/run-tests.sh') + if (it.key.startsWith('dtest')) { + it.value.put('python-dtest', true) + } +} + +def stepsMap = buildSteps + testSteps + +// define matrix axes +def Map matrix_axes = [ + arch: archsSupported, + jdk: jdksSupported, + python: pythonsSupported, + cython: ['yes', 'no'], + step: stepsMap.keySet(), + split: (1..64).toList() // needs to be max splits can be ?? +] + +def List _axes = getMatrixAxes(matrix_axes).findAll { axis -> + (isArchEnabled(axis['arch'])) && // skip disabled archs + (isJdkEnabled(axis['jdk'])) && // skip disabled jdks + (isStageEnabled(axis['step'])) && // skip disabled steps + !(axis['python'] != pythonDefault && 'cqlsh-test' != axis['step']) && // Use only python 3.8 for all tests but cqlsh-test + !(axis['cython'] != 'no' && 'cqlsh-test' != axis['step']) && // cython only for cqlsh-test, disable for others + !(axis['jdk'] != jdkDefault && 'cqlsh-test' == axis['step']) && // run cqlsh-test only with jdk11 + // Disable splits for all but proper stages + !(axis['split'] > 1 && !stepsMap.findAll { entry -> entry.value.splits >= axis['split'] }.keySet().contains(axis['step'])) && + // run only the build types on non-amd64 + !(axis['arch'] != 'amd64' && stepsMap.findAll { entry -> 'build' == entry.value.type }.keySet().contains(axis['step'])) +} + +// Prepare tasks +def Map tasks = [ + jars: [failFast: !isPostCommit()], + tests: [failFast: !isPostCommit()], +] +for (def axis in _axes) { + def cell = axis + def name = getStepName(cell, stepsMap[cell.step]) + tasks[cell.step == "jar" ? "jars" : "tests"][name] = { -> + "${stepsMap[cell.step].type}"(stepsMap[cell.step], cell) // build() or test() + } +} + pipeline { - agent { label 'cassandra' } + agent none + options { + skipDefaultCheckout() + + // FIXME problem with this retry approach is it retries the whole pipeline. what we want is just cell retry + //retry(2) + } + parameters { + string(name: 'repository', defaultValue: scm.userRemoteConfigs[0].url, description: 'Cassandra Repository') + string(name: 'branch', defaultValue: env.BRANCH_NAME, description: 'Branch') + choice(name: 'architecture', choices: archsSupported + "all", description: 'Pick architecture. The ARM64 is disabled by default at the moment.') + choice(name: 'jdk', choices: jdksSupported + "all", description: 'Pick JDK versions.') + booleanParam(name: 'stage_artifacts', defaultValue: false, description: 'Disable to exlude stage') // return default to true + booleanParam(name: 'stage_lint', defaultValue: false) // return default to true + booleanParam(name: 'stage_debian', defaultValue: false) // return default to true + booleanParam(name: 'stage_redhat', defaultValue: false) // return default to true + booleanParam(name: 'stage_fqltool-test', defaultValue: false) // return default to true + booleanParam(name: 'stage_cqlsh-test', defaultValue: false) // return default to true + booleanParam(name: 'stage_test-cdc', defaultValue: false) // return default to true + booleanParam(name: 'stage_test', defaultValue: false) // return default to true + booleanParam(name: 'stage_test-trie', defaultValue: false) // return default to true + booleanParam(name: 'stage_test-compression', defaultValue: false) // return default to true + booleanParam(name: 'stage_stress-test', defaultValue: false) // return default to true + booleanParam(name: 'stage_test-burn', defaultValue: false) // return default to true + booleanParam(name: 'stage_long-test', defaultValue: false) // return default to true + booleanParam(name: 'stage_test-oa', defaultValue: false) // return default to true + booleanParam(name: 'stage_test-system-keyspace-directory', defaultValue: false) // return default to true + booleanParam(name: 'stage_jvm-dtest', defaultValue: false) // return default to true + booleanParam(name: 'stage_jvm-dtest-upgrade', defaultValue: false) // return default to true + booleanParam(name: 'stage_simulator-dtest', defaultValue: false) // return default to true + booleanParam(name: 'stage_dtest', defaultValue: false) // return default to true + booleanParam(name: 'stage_dtest-novnode', defaultValue: false) // return default to true + booleanParam(name: 'stage_dtest-offheap', defaultValue: false) // return default to true + booleanParam(name: 'stage_dtest-large', defaultValue: false) // return default to true + booleanParam(name: 'stage_dtest-large-novnode', defaultValue: false) // return default to true + booleanParam(name: 'stage_dtest-upgrade', defaultValue: false) // return default to true + booleanParam(name: 'stage_dtest-upgrade-novnode', defaultValue: false) // return default to true + booleanParam(name: 'stage_dtest-upgrade-large', defaultValue: false) // return default to true + booleanParam(name: 'stage_dtest-upgrade-novnode-large', defaultValue: false) // return default to true + } + environment { + javaVersionsSupported = jdksSupported.join(',') + javaVersionDefault = "${jdkDefault}" + } stages { - stage('Init') { + stage('jar') { steps { - cleanWs() - script { - currentBuild.result='SUCCESS' - } + script { + parallel(tasks['jars']) + } } } - stage('Build') { + stage('Tests') { + when { + expression { tasks['tests'].size() > 1 } // Skip if empty (failfast counts as an element) + } steps { - script { - def attempt = 1 - retry(2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - build job: "${env.JOB_NAME}-artifacts" + script { + parallel(tasks['tests']) } - } } } - stage('Test') { - parallel { - stage('stress') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - stress = build job: "${env.JOB_NAME}-stress-test", propagate: false - if (stress.result != 'FAILURE') break - } - if (stress.result != 'SUCCESS') unstable('stress test failures') - if (stress.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('stress-test', stress.getNumber()) - } - } - } - } - } - stage('fqltool') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - fqltool = build job: "${env.JOB_NAME}-fqltool-test", propagate: false - if (fqltool.result != 'FAILURE') break - } - if (fqltool.result != 'SUCCESS') unstable('fqltool test failures') - if (fqltool.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('fqltool-test', fqltool.getNumber()) - } - } - } - } - } - stage('units') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - test = build job: "${env.JOB_NAME}-test", propagate: false - if (test.result != 'FAILURE') break - } - if (test.result != 'SUCCESS') unstable('unit test failures') - if (test.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test', test.getNumber()) - } - } - } - } - } - stage('long units') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - long_test = build job: "${env.JOB_NAME}-long-test", propagate: false - if (long_test.result != 'FAILURE') break - } - if (long_test.result != 'SUCCESS') unstable('long unit test failures') - if (long_test.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('long-test', long_test.getNumber()) - } - } - } - } - } - stage('burn') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - burn = build job: "${env.JOB_NAME}-test-burn", propagate: false - if (burn.result != 'FAILURE') break - } - if (burn.result != 'SUCCESS') unstable('burn test failures') - if (burn.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test-burn', burn.getNumber()) - } - } - } + stage('Summary') { + steps { + script { + if(stageResults.find { "FAILURE" == it.value }) { + currentBuild.result='FAILURE' + error("Build failed due to failed stages") + } else if(stageResults.find { "UNSTABLE" == it.value || "ABORTED" == it.value }) { + if ("ABORTED" != currentBuild.result) { currentBuild.result='UNSTABLE' } + echo("Build unstable due to unstable or aborted stages") Review Comment: on a failed/aborted build we're getting ``` MissingPropertyException: No such property: stageResults for class: groovy.lang.Binding ``` but I'm not sure where it's coming from… see https://ci-cassandra.apache.org/job/Cassandra-5-devbranch/59/console -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
