This is an automated email from the ASF dual-hosted git repository. kwin pushed a commit to branch feature/parallel-builds in repository https://gitbox.apache.org/repos/asf/sling-tooling-jenkins.git
commit 56281bc3be48f0a0f3134a8b7e5c874a47ebf2e3 Author: Konrad Windszus <[email protected]> AuthorDate: Wed Jun 15 09:47:47 2022 +0200 SLING-9948 parallelize steps reuse build artifacts for SonarCloud analysis deploy only if every other previous stage was successfully executed --- vars/slingOsgiBundleBuild.groovy | 204 ++++++++++++++++++++++++++------------- 1 file changed, 137 insertions(+), 67 deletions(-) diff --git a/vars/slingOsgiBundleBuild.groovy b/vars/slingOsgiBundleBuild.groovy index 45ede41..875ef3d 100644 --- a/vars/slingOsgiBundleBuild.groovy +++ b/vars/slingOsgiBundleBuild.groovy @@ -9,71 +9,51 @@ def call(Map params = [:]) { githubCredentialsId: 'sling-github-token' ] - node(globalConfig.mainNodeLabel) { - - def helper = new SlingJenkinsHelper() - - helper.runWithErrorHandling({ jobConfig -> - if ( jobConfig.enabled ) { - - // the reference build is always the first one, and the only one to deploy, archive artifacts, etc - // usually this is the build done with the oldest JDK version, to ensure maximum compatibility - def isReferenceStage = true - - jobConfig.jdks.each { jdkVersion -> - stageDefinition = defineStage(globalConfig, jobConfig, jdkVersion, isReferenceStage) - stageDefinition.call() - isReferenceStage = false - currentBuild.result = "SUCCESS" + def helper = new SlingJenkinsHelper() + + helper.runWithErrorHandling({ jobConfig -> + if ( jobConfig.enabled ) { + + // do a quick sanity check first without tests + // mvn clean compile + node(globalConfig.mainNodeLabel) { + stage("Sanity Check") { + withMaven(maven: globalConfig.mvnVersion, + jdk: jenkinsJdkLabel(8, globalConfig), // TODO: use Java version from first Job Config + publisherStrategy: 'EXPLICIT') ) { + sh "mvn clean compile ${additionalMavenParams}" + } } + } - // this might fail if there are no jdks defined, but that's always an error - // also, we don't activate any Maven publisher since we don't want this part of the - // build tracked, but using withMaven(...) allows us to easily reuse the same - // Maven and JDK versions - def additionalMavenParams = additionalMavenParams(jobConfig) - def isPrBuild = env.BRANCH_NAME.startsWith("PR-") - - if ( jobConfig.sonarQubeEnabled ) { - stage('SonarCloud') { - // As we don't have the global SonarCloud conf for now, we can't use #withSonarQubeEnv so we need to set the following props manually - def sonarcloudParams="-Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_${jobConfig.repoName} -Pjacoco-report -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco-merged/jacoco.xml ${jobConfig.sonarQubeAdditionalParams}" - if ( jobConfig.sonarQubeUseAdditionalMavenParams ) { - sonarcloudParams="${sonarcloudParams} ${additionalMavenParams}" - } - // Params are different if it's a PR or if it's not - // Note: soon we won't have to handle that manually, see https://jira.sonarsource.com/browse/SONAR-11853 - if ( isPrBuild ) { - sonarcloudParams="${sonarcloudParams} -Dsonar.pullrequest.branch=${CHANGE_BRANCH} -Dsonar.pullrequest.base=${CHANGE_TARGET} -Dsonar.pullrequest.key=${CHANGE_ID}" - } else if ( env.BRANCH_NAME != "master" ) { - sonarcloudParams="${sonarcloudParams} -Dsonar.branch.name=${BRANCH_NAME}" - } - static final String SONAR_PLUGIN_GAV = 'org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184' - // Alls params are set, let's execute using #withCrendentials to hide and mask Robert's token - withCredentials([string(credentialsId: 'sonarcloud-token-rombert', variable: 'SONAR_TOKEN')]) { - // always build with Java 11 (that is the minimum version supported: https://sonarcloud.io/documentation/appendices/end-of-support/) - withMaven(maven: globalConfig.mvnVersion, - jdk: jenkinsJdkLabel(11, globalConfig), - publisherStrategy: 'EXPLICIT') { - try { - sh "mvn -U clean verify ${SONAR_PLUGIN_GAV}:sonar ${sonarcloudParams} -Pci" - } catch ( Exception e ) { - // TODO - we should check the actual failure cause here, but see - // https://stackoverflow.com/questions/55742773/get-the-cause-of-a-maven-build-failure-inside-a-jenkins-pipeline/55744122 - echo "Marking build unstable due to mvn sonar:sonar failing. See https://cwiki.apache.org/confluence/display/SLING/SonarCloud+analysis for more info." - currentBuild.result = 'UNSTABLE' - } - } - } + // the reference build is always the first one, and the only one to deploy, archive artifacts, etc + // usually this is the build done with the oldest JDK version, to ensure maximum compatibility + def isReferenceStage = true + + // contains the label as key and a closure to execute as value + def stepsMap = [:] + // parallel execution of all build jobs (reference potentially including SonarQube) + jobConfig.jdks.each { jdkVersion -> + stageDefinition = defineStage(globalConfig, jobConfig, jdkVersion, isReferenceStage) + stepsMap["$jdkVersion"] = stageDefinition + isReferenceStage = false + currentBuild.result = "SUCCESS" + } + + parallel stepsMap + + // last stage is deploy + if ( shouldDeploy() ) { + node(globalConfig.mainNodeLabel) { + stage("Deploy to Nexus") { + deployToNexus() } - } else { - echo "SonarQube execution is disabled" } - } else { - echo "Job is disabled, not building" } - }) - } + } else { + echo "Job is disabled, not building" + } + }) } def jenkinsJdkLabel(int jdkVersion, def globalConfig) { @@ -105,7 +85,15 @@ def defineStage(def globalConfig, def jobConfig, def jdkVersion, def isReference if ( notMaster || !isSnapshot ) { goal = "verify" echo "Maven goal set to ${goal} since branch is not master ( ${env.BRANCH_NAME} ) or version is not snapshot ( ${mavenVersion} )" - } + } else { + String localRepoPath = "${env.WORKSPACE}/local-snapshots-dir" + // Make sure the directory is wiped. + dir(localRepoPath) { + deleteDir() + } + // main build with IT for properly calculating coverage + additionalMavenParams = "${additionalMavenParams} -DaltDeploymentRepository=snapshot-repo::default::file:${localRepoPath}" + } } def invocation = { @@ -122,21 +110,103 @@ def defineStage(def globalConfig, def jobConfig, def jdkVersion, def isReference if ( isReferenceStage && jobConfig.archivePatterns ) { archiveArtifacts(artifacts: SlingJenkinsHelper.jsonArrayToCsv(jobConfig.archivePatterns), allowEmptyArchive: true) } + if ( isReferenceStage && shouldDeploy ) { + // Stash the build results so we can deploy them on another node + stash name: 'local-snapshots-dir', includes: 'local-snapshots-dir/**' + } } def branchConfig = jobConfig?.branches?."$env.BRANCH_NAME" ?: [:] - if ( branchConfig.nodeLabel && branchConfig.nodeLabel != globalConfig.mainNodeLabel ) - invocation = wrapInNode(invocation,branchConfig.nodeLabel) - return { - stage("Build (Java ${jdkVersion}, ${goal})") { - invocation.call() + wrapInNode(branchConfig.nodeLabel && branchConfig.nodeLabel != globalConfig.mainNodeLabel), { + stage("Build (Java ${jdkVersion}, ${goal})") { + invocation.call() + } + if ( isReferenceStage ) { + // SonarQube (must be wrapped in same node) + if ( jobConfig.sonarQubeEnabled ) { + stage('Analyse with SonarCloud') { + timeout(60) { + analyseWithSonarCloud(globalConfig) + } + } + } + } + } + } +} + +def analyseWithSonarCloud(def globalConfig) { + // this might fail if there are no jdks defined, but that's always an error + // also, we don't activate any Maven publisher since we don't want this part of the + // build tracked, but using withMaven(...) allows us to easily reuse the same + // Maven and JDK versions + def additionalMavenParams = additionalMavenParams(jobConfig) + def isPrBuild = env.BRANCH_NAME.startsWith("PR-") + + // As we don't have the global SonarCloud conf for now, we can't use #withSonarQubeEnv so we need to set the following props manually + def sonarcloudParams="-Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_${jobConfig.repoName} -Pjacoco-report -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco-merged/jacoco.xml ${jobConfig.sonarQubeAdditionalParams}" + if ( jobConfig.sonarQubeUseAdditionalMavenParams ) { + sonarcloudParams="${sonarcloudParams} ${additionalMavenParams}" + } + // Params are different if it's a PR or if it's not + // Note: soon we won't have to handle that manually, see https://jira.sonarsource.com/browse/SONAR-11853 + if ( isPrBuild ) { + sonarcloudParams="${sonarcloudParams} -Dsonar.pullrequest.branch=${CHANGE_BRANCH} -Dsonar.pullrequest.base=${CHANGE_TARGET} -Dsonar.pullrequest.key=${CHANGE_ID}" + } else if ( env.BRANCH_NAME != "master" ) { + sonarcloudParams="${sonarcloudParams} -Dsonar.branch.name=${BRANCH_NAME}" + } + static final String SONAR_PLUGIN_GAV = 'org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184' + // Alls params are set, let's execute using #withCrendentials to hide and mask Robert's token + withCredentials([string(credentialsId: 'sonarcloud-token-rombert', variable: 'SONAR_TOKEN')]) { + // always build with Java 11 (that is the minimum version supported: https://sonarcloud.io/documentation/appendices/end-of-support/) + withMaven(maven: globalConfig.mvnVersion, + jdk: jenkinsJdkLabel(11, globalConfig), + publisherStrategy: 'EXPLICIT') { + try { + sh "mvn ${SONAR_PLUGIN_GAV}:sonar ${sonarcloudParams}" + } catch ( Exception e ) { + // TODO - we should check the actual failure cause here, but see + // https://stackoverflow.com/questions/55742773/get-the-cause-of-a-maven-build-failure-inside-a-jenkins-pipeline/55744122 + echo "Marking build unstable due to mvn sonar:sonar failing. See https://cwiki.apache.org/confluence/display/SLING/SonarCloud+analysis for more info." + currentBuild.result = 'UNSTABLE' + } + } + } +} + +def deployToNexus() { + node('nexus-deploy') { + timeout(60) { + echo "Running on node ${env.NODE_NAME}" + // first clear workspace + deleteDir() + // Nexus deployment needs pom.xml + checkout scm + // Unstash the previously stashed build results. + unstash name: 'local-snapshots-dir' + // https://www.mojohaus.org/wagon-maven-plugin/merge-maven-repos-mojo.html + String mavenArguments = "${wagonPluginGav}:merge-maven-repos -Dwagon.target=https://repository.apache.org/content/repositories/snapshots -Dwagon.targetId=apache.snapshots.https -Dwagon.source=file:${env.WORKSPACE}/local-snapshots-dir" + pipelineSupport.executeMaven(this, mavenArguments, false) } } } -def wrapInNode(Closure invocation, def nodeLabel) { +boolean shouldDeploy() { + // check branch name + if ( !isOnMainBranch() ) { + return false; + } + // check maven goals + // check version +} + +boolean isOnMainBranch() { + return env.BRANCH_NAME == "master" +} + +def wrapInNode(def nodeLabel, Closure invocation) { return { node(nodeLabel) { checkout scm
