michaelsembwever commented on code in PR #2852: URL: https://github.com/apache/cassandra/pull/2852#discussion_r1545317676
########## .jenkins/Jenkinsfile: ########## @@ -11,762 +11,534 @@ // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// Se# Licensed to the Apache Software Foundation (ASF) under onee the License for the specific language governing permissions and +// See the License for the specific language governing permissions and // limitations under the License. // // -// Jenkins declaration of how to build and test the current codebase. -// Jenkins infrastructure related settings should be kept in -// https://github.com/apache/cassandra-builds/blob/trunk/jenkins-dsl/cassandra_job_dsl_seed.groovy +// Jenkins CI declaration. +// +// The declarative pipeline is presented first as a high level view. +// +// Build and Test Stages are dynamic, the full possible list defined by the `tasks()` function. +// There is a choice of pipeline profles with sets of tasks that are run, see `pipelineProfiles()`. +// +// All tasks use the dockerised CI-agnostic scripts found under `.build/docker/` +// The `type: test` always `.build/docker/run-tests.sh` +// +// +// This Jenkinsfile is expected to work on any Jenkins infrastructure. +// The controller should have 4 cpu, 12GB ram (and be configured to use `-XX:+UseG1GC -Xmx8G`) +// It is required to have agents providing five labels, each that can provide docker and the following capabilities: +// - cassandra-amd64-small : 1 cpu, 1GB ram +// - cassandra-small : 1 cpu, 1GB ram (alias for above but for any arch) +// - cassandra-amd64-medium : 3 cpu, 5GB ram +// - cassandra-medium : 3 cpu, 5GB ram (alias for above but for any arch) +// - cassandra-amd64-large : 7 cpu, 14GB ram +// +// When running builds parameterised to other architectures the corresponding labels are expected. +// For example 'arm64' requires the labels: cassandra-arm64-small, cassandra-arm64-medium, cassandra-arm64-large. +// +// Plugins required are: +// git, workflow-job, workflow-cps, junit, workflow-aggregator, ws-cleanup, pipeline-build-step, test-stability, copyartifact. +// +// Any functionality that depends upon ASF Infra ( i.e. the canonical ci-cassandra.a.o ) +// will be ignored when run on other environments. +// // // Validate/lint this file using the following command // `curl -X POST -F "jenkinsfile=<.jenkins/Jenkinsfile" https://ci-cassandra.apache.org/pipeline-model-converter/validate` +// + pipeline { - agent { label 'cassandra' } + agent { label 'cassandra-small||cassandra-amd64-small' } + parameters { + string(name: 'repository', defaultValue: scm.userRemoteConfigs[0].url, description: 'Cassandra Repository') + string(name: 'branch', defaultValue: env.BRANCH_NAME, description: 'Branch') + + choice(name: 'profile', choices: pipelineProfiles().keySet() as List, description: 'Pick a pipeline profile.') + string(name: 'profile_custom_regexp', defaultValue: '', description: 'Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: stress.*|jvm-dtest.*') + + choice(name: 'architecture', choices: archsSupported() + "all", description: 'Pick architecture. The ARM64 is disabled by default at the moment.') + string(name: 'jdk', defaultValue: "", description: 'Restrict JDK versions. (e.g. "11", "17", etc)') + + string(name: 'dtest_repository', defaultValue: 'https://github.com/apache/cassandra-dtest' ,description: 'Cassandra DTest Repository') + string(name: 'dtest_branch', defaultValue: 'trunk', description: 'DTest Branch') + } stages { - stage('Init') { + stage('jar') { steps { - cleanWs() - script { - currentBuild.result='SUCCESS' - } + script { + Map jars = tasks()['jars'] + assertJarTasks(jars) + parallel(jars) + } } } - stage('Build') { + stage('Tests') { + when { + expression { hasNonJarTasks() } + } steps { - script { - def attempt = 1 - retry(2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - build job: "${env.JOB_NAME}-artifacts" + script { + parallel(tasks()['tests']) } - } } } - stage('Test') { - parallel { - stage('stress') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - stress = build job: "${env.JOB_NAME}-stress-test", propagate: false - if (stress.result != 'FAILURE') break - } - if (stress.result != 'SUCCESS') unstable('stress test failures') - if (stress.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('stress-test', stress.getNumber()) - } - } - } - } - } - stage('fqltool') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - fqltool = build job: "${env.JOB_NAME}-fqltool-test", propagate: false - if (fqltool.result != 'FAILURE') break - } - if (fqltool.result != 'SUCCESS') unstable('fqltool test failures') - if (fqltool.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('fqltool-test', fqltool.getNumber()) - } - } - } - } - } - stage('units') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - test = build job: "${env.JOB_NAME}-test", propagate: false - if (test.result != 'FAILURE') break - } - if (test.result != 'SUCCESS') unstable('unit test failures') - if (test.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test', test.getNumber()) - } - } - } - } - } - stage('long units') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - long_test = build job: "${env.JOB_NAME}-long-test", propagate: false - if (long_test.result != 'FAILURE') break - } - if (long_test.result != 'SUCCESS') unstable('long unit test failures') - if (long_test.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('long-test', long_test.getNumber()) - } - } - } - } - } - stage('burn') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - burn = build job: "${env.JOB_NAME}-test-burn", propagate: false - if (burn.result != 'FAILURE') break - } - if (burn.result != 'SUCCESS') unstable('burn test failures') - if (burn.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test-burn', burn.getNumber()) - } - } - } - } - } - stage('cdc') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - cdc = build job: "${env.JOB_NAME}-test-cdc", propagate: false - if (cdc.result != 'FAILURE') break - } - if (cdc.result != 'SUCCESS') unstable('cdc failures') - if (cdc.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test-cdc', cdc.getNumber()) - } - } - } - } - } - stage('compression') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - compression = build job: "${env.JOB_NAME}-test-compression", propagate: false - if (compression.result != 'FAILURE') break - } - if (compression.result != 'SUCCESS') unstable('compression failures') - if (compression.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test-compression', compression.getNumber()) - } - } - } - } - } - stage('oa') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - oa = build job: "${env.JOB_NAME}-test-oa", propagate: false - if (oa.result != 'FAILURE') break - } - if (oa.result != 'SUCCESS') unstable('oa failures') - if (oa.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test-oa', oa.getNumber()) - } - } - } - } - } - stage('system-keyspace-directory') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - system_keyspace_directory = build job: "${env.JOB_NAME}-test-system-keyspace-directory", propagate: false - if (system_keyspace_directory.result != 'FAILURE') break - } - if (system_keyspace_directory.result != 'SUCCESS') unstable('system-keyspace-directory failures') - if (system_keyspace_directory.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test-system-keyspace-directory', system_keyspace_directory.getNumber()) - } - } - } - } - } - stage('latest') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - latest = build job: "${env.JOB_NAME}-test-latest", propagate: false - if (latest.result != 'FAILURE') break - } - if (latest.result != 'SUCCESS') unstable('test-latest failures') - if (latest.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('test-latest', latest.getNumber()) - } - } - } - } - } - stage('cqlsh') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - cqlsh = build job: "${env.JOB_NAME}-cqlsh-tests", propagate: false - if (cqlsh.result != 'FAILURE') break - } - if (cqlsh.result != 'SUCCESS') unstable('cqlsh failures') - if (cqlsh.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('cqlsh-tests', cqlsh.getNumber()) - } - } - } - } - } - stage('simulator-dtest') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - simulator_dtest = build job: "${env.JOB_NAME}-simulator-dtest", propagate: false - if (simulator_dtest.result != 'FAILURE') break - } - if (simulator_dtest.result != 'SUCCESS') unstable('simulator-dtest failures') - if (simulator_dtest.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('simulator-dtest', simulator_dtest.getNumber()) - } - } - } - } - } + stage('Summary') { + steps { + generateTestReports() } } - stage('Distributed Test') { - parallel { - stage('jvm-dtest') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - jvm_dtest = build job: "${env.JOB_NAME}-jvm-dtest", propagate: false - if (jvm_dtest.result != 'FAILURE') break - } - if (jvm_dtest.result != 'SUCCESS') unstable('jvm-dtest failures') - if (jvm_dtest.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('jvm-dtest', jvm_dtest.getNumber()) - } - } - } - } - } - stage('jvm-dtest-novnode') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - jvm_dtest_novnode = build job: "${env.JOB_NAME}-jvm-dtest-novnode", propagate: false - if (jvm_dtest_novnode.result != 'FAILURE') break - } - if (jvm_dtest_novnode.result != 'SUCCESS') unstable('jvm-dtest-novnode failures') - if (jvm_dtest_novnode.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('jvm-dtest-novnode', jvm_dtest_novnode.getNumber()) - } - } - } - } - } - stage('jvm-dtest-upgrade') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - jvm_dtest_upgrade = build job: "${env.JOB_NAME}-jvm-dtest-upgrade", propagate: false - if (jvm_dtest_upgrade.result != 'FAILURE') break - } - if (jvm_dtest_upgrade.result != 'SUCCESS') unstable('jvm-dtest-upgrade failures') - if (jvm_dtest_upgrade.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('jvm-dtest-upgrade', jvm_dtest_upgrade.getNumber()) - } - } - } - } - } - stage('jvm-dtest-upgrade-novnode') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - jvm_dtest_upgrade_novnode = build job: "${env.JOB_NAME}-jvm-dtest-upgrade-novnode", propagate: false - if (jvm_dtest_upgrade_novnode.result != 'FAILURE') break - } - if (jvm_dtest_upgrade_novnode.result != 'SUCCESS') unstable('jvm-dtest-upgrade-novnode failures') - if (jvm_dtest_upgrade_novnode.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('jvm-dtest-upgrade-novnode', jvm_dtest_upgrade_novnode.getNumber()) - } - } - } - } - } - stage('dtest') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest = build job: "${env.JOB_NAME}-dtest", propagate: false - if (dtest.result != 'FAILURE') break - } - if (dtest.result != 'SUCCESS') unstable('dtest failures') - if (dtest.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest', dtest.getNumber()) - } - } - } - } - } - stage('dtest-large') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_large = build job: "${env.JOB_NAME}-dtest-large", propagate: false - if (dtest_large.result != 'FAILURE') break - } - if (dtest_large.result != 'SUCCESS') unstable('dtest-large failures') - if (dtest_large.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-large', dtest_large.getNumber()) - } - } - } - } - } - stage('dtest-novnode') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_novnode = build job: "${env.JOB_NAME}-dtest-novnode", propagate: false - if (dtest_novnode.result != 'FAILURE') break - } - if (dtest_novnode.result != 'SUCCESS') unstable('dtest-novnode failures') - if (dtest_novnode.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-novnode', dtest_novnode.getNumber()) - } - } - } - } - } - stage('dtest-offheap') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_offheap = build job: "${env.JOB_NAME}-dtest-offheap", propagate: false - if (dtest_offheap.result != 'FAILURE') break - } - if (dtest_offheap.result != 'SUCCESS') unstable('dtest-offheap failures') - if (dtest_offheap.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-offheap', dtest_offheap.getNumber()) - } - } - } - } - } - stage('dtest-large-novnode') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_large_novnode = build job: "${env.JOB_NAME}-dtest-large-novnode", propagate: false - if (dtest_large_novnode.result != 'FAILURE') break - } - if (dtest_large_novnode.result != 'SUCCESS') unstable('dtest-large-novnode failures') - if (dtest_large_novnode.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-large-novnode', dtest_large_novnode.getNumber()) - } - } - } - } - } - stage('dtest-upgrade') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_upgrade = build job: "${env.JOB_NAME}-dtest-upgrade", propagate: false - if (dtest_upgrade.result != 'FAILURE') break - } - if (dtest_upgrade.result != 'SUCCESS') unstable('dtest failures') - if (dtest_upgrade.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-upgrade', dtest_upgrade.getNumber()) - } - } - } - } - } - stage('dtest-upgrade-large') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_upgrade = build job: "${env.JOB_NAME}-dtest-upgrade-large", propagate: false - if (dtest_upgrade.result != 'FAILURE') break - } - if (dtest_upgrade.result != 'SUCCESS') unstable('dtest failures') - if (dtest_upgrade.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-upgrade', dtest_upgrade.getNumber()) - } - } - } - } - } - stage('dtest-upgrade-novnode') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_upgrade_novnode = build job: "${env.JOB_NAME}-dtest-upgrade-novnode", propagate: false - if (dtest_upgrade_novnode.result != 'FAILURE') break - } - if (dtest_upgrade_novnode.result != 'SUCCESS') unstable('dtest-upgrade-novnode failures') - if (dtest_upgrade_novnode.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-upgrade-novnode', dtest_upgrade_novnode.getNumber()) - } - } - } - } - } - stage('dtest-upgrade-novnode-large') { - steps { - script { - def attempt = 1 - while (attempt <=2) { - if (attempt > 1) { - sleep(60 * attempt) - } - attempt = attempt + 1 - dtest_upgrade_novnode_large = build job: "${env.JOB_NAME}-dtest-upgrade-novnode-large", propagate: false - if (dtest_upgrade_novnode_large.result != 'FAILURE') break - } - if (dtest_upgrade_novnode_large.result != 'SUCCESS') unstable('dtest-upgrade-novnode-large failures') - if (dtest_upgrade_novnode_large.result == 'FAILURE') currentBuild.result='FAILURE' - } - } - post { - always { - warnError('missing test xml files') { - script { - copyTestResults('dtest-upgrade-novnode-large', dtest_upgrade_novnode_large.getNumber()) - } - } - } - } - } + } + post { + always { + sendNotifications() + } + } +} + +/////////////////////////// +//// scripting support //// +/////////////////////////// + +def archsSupported() { return ["amd64", "arm64"] } +def pythonsSupported() { return ["3.8", "3.11"] } +def pythonDefault() { return "3.8" } + +def pipelineProfiles() { + return [ + 'packaging': ['artifacts', 'lint', 'debian', 'redhat'], + 'skinny': ['lint', 'cqlsh-test', 'test', 'jvm-dtest', 'simulator-dtest', 'dtest'], + 'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest'], + 'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade'], + 'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-novnode-large'], + 'custom': [] + ] +} + +def tasks() { + // Steps config + def buildSteps = [ + 'jar': [script: 'build-jars.sh', toCopy: null], + 'artifacts': [script: 'build-artifacts.sh', toCopy: 'apache-cassandra-*.tar.gz,apache-cassandra-*.jar,apache-cassandra-*.pom'], + 'lint': [script: 'check-code.sh', toCopy: null], + 'debian': [script: 'build-debian.sh', toCopy: 'cassandra_*,cassandra-tools_*'], + 'redhat': [script: 'build-redhat.sh rpm', toCopy: '*.rpm'], + ] + buildSteps.each() { + it.value.put('type', 'build') + it.value.put('size', 'small') + it.value.put('splits', 1) + } + + def testSteps = [ Review Comment: agree. this is to be addressed in [CASSANDRA-18731](https://issues.apache.org/jira/browse/CASSANDRA-18731) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
