This is an automated email from the ASF dual-hosted git repository. rantunes pushed a commit to branch kie-issues_821 in repository https://gitbox.apache.org/repos/asf/incubator-kie-kogito-images.git
commit 6ce8d17cc20f02a5ac04d82e5714e6f2f881397d Author: Rodrigo Antunes <[email protected]> AuthorDate: Mon Jan 29 16:12:12 2024 -0300 Kogito Images weekly job --- .ci/jenkins/Jenkinsfile.weekly.deploy | 359 ++++++++++++++++++++++++++++++++++ .ci/jenkins/dsl/jobs.groovy | 53 +++++ 2 files changed, 412 insertions(+) diff --git a/.ci/jenkins/Jenkinsfile.weekly.deploy b/.ci/jenkins/Jenkinsfile.weekly.deploy new file mode 100644 index 00000000..1d1fcc23 --- /dev/null +++ b/.ci/jenkins/Jenkinsfile.weekly.deploy @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@Library('jenkins-pipeline-shared-libraries')_ + +deployProperties = [:] + +changesDone = false + +BUILT_IMAGES = Collections.synchronizedList([]) +BUILD_FAILED_IMAGES = Collections.synchronizedList([]) +TEST_FAILED_IMAGES = Collections.synchronizedList([]) + +pipeline { + agent { + docker { + image env.AGENT_DOCKER_BUILDER_IMAGE + args env.AGENT_DOCKER_BUILDER_ARGS + } + } + + options { + timeout(time: 240, unit: 'MINUTES') + skipDefaultCheckout() + } + + environment { + KOGITO_CI_EMAIL_TO = credentials("${JENKINS_EMAIL_CREDS_ID}") + } + + stages { + stage('Initialization') { + steps { + script { + util.waitForDocker() + currentBuild.displayName = params.DISPLAY_NAME ?: currentBuild.displayName + + dir(getRepoName()) { + checkoutRepo() + } + + env.PROJECT_VERSION = sh(returnStdout: true, script: "python scripts/retrieve_version.py").trim() + } + } + post { + always { + script { + setDeployPropertyIfNeeded('git.branch', getBuildBranch()) + setDeployPropertyIfNeeded('git.author', getGitAuthor()) + setDeployPropertyIfNeeded('project.version', getProjectVersion()) + } + } + } + } + + stage('Update project version') { + steps { + script { + dir(getRepoName()) { + versionCmd = "python scripts/manage-kogito-version.py --bump-to ${getProjectVersion()} --confirm" + if (getBuildBranch() != 'main') { + versionCmd += " --examples-ref 'nightly-'${getBuildBranch()}" + } + sh versionCmd + } + } + } + } + + stage('Validate CeKit Image and Modules descriptors') { + steps { + script { + dir(getRepoName()) { + sh ''' + curl -Ls https://github.com/kiegroup/kie-cloud-tools/releases/download/v1.3.6/cekit-image-validator-runner.tgz --output cekit-image-validator-runner.tgz + tar -xzvf cekit-image-validator-runner.tgz + chmod +x cekit-image-validator-runner + ''' + sh './cekit-image-validator-runner modules/' + getImages().each { image -> sh "./cekit-image-validator-runner ${image}-image.yaml" } + } + } + } + } + + stage('Build, Push & Test Images') { + steps { + script { + dir(getRepoName()) { + parallelStages = [:] + getImages().each { image -> + parallelStages["Build&Test ${image}"] = createBuildAndTestStageClosure(image) + } + parallel parallelStages + } + } + } + post { + always { + script { + // Store image deployment information + String imgPrefix = "${getRepoName()}.image" + setDeployPropertyIfNeeded("${imgPrefix}.registry", getDeployImageRegistry()) + setDeployPropertyIfNeeded("${imgPrefix}.namespace", getDeployImageNamespace()) + setDeployPropertyIfNeeded("${imgPrefix}.names", getBuiltImages().join(',')) + setDeployPropertyIfNeeded("${imgPrefix}.name-suffix", getDeployImageNameSuffix()) + setDeployPropertyIfNeeded("${imgPrefix}.tag", getDeployImageTag()) + } + } + } + } + + stage('Create and push a new tag') { + steps { + script { + projectVersion = getProjectVersion(false) + dir(getRepoName()) { + githubscm.setUserConfigFromCreds(getGitAuthorPushCredsId()) + githubscm.tagRepository(projectVersion) + githubscm.pushRemoteTag('origin', projectVersion, getGitAuthorPushCredsId()) + } + } + } + } + + stage('Finalize') { + steps { + script { + if (getBuildFailedImages()) { + currentBuild.currentResult = 'FAILURE' + } + } + } + } + } + post { + always { + script { + def propertiesStr = deployProperties.collect { entry -> "${entry.key}=${entry.value}" }.join('\n') + writeFile(file : env.PROPERTIES_FILE_NAME , text : propertiesStr) + archiveArtifacts artifacts: env.PROPERTIES_FILE_NAME, allowEmptyArchive:true + } + } + cleanup { + cleanWs() + } + unsuccessful { + sendUnsuccessfulNotification() + } + } +} + +void sendUnsuccessfulNotification() { + if (params.SEND_NOTIFICATION) { + mailer.sendMarkdownTestSummaryNotification('Deploy', getNotificationSubject(), [env.KOGITO_CI_EMAIL_TO]) + } else { + echo 'No notification sent per configuration' + } +} + +void sendNotification(String body) { + emailext body: body, + subject: getNotificationSubject(), + to: env.KOGITO_CI_EMAIL_TO +} + +String getNotificationSubject() { + return "[${getBuildBranch()}] Kogito Images" +} + +void checkoutRepo() { + deleteDir() + checkout(githubscm.resolveRepository(getRepoName(), getGitAuthor(), getBuildBranch(), false, getGitAuthorCredsId())) +} + +void commitAndPushChanges(String commitMsg) { + githubscm.setUserConfigFromCreds(getGitAuthorPushCredsId()) + githubscm.commitChanges(commitMsg) + githubscm.pushObject('origin', getPRBranch(), getGitAuthorPushCredsId()) + changesDone = true +} + +void createBuildAndTestStageClosure(String image) { + return { + stage("Build&Test ${image}") { + List buildParams = [] + buildParams.add(string(name: 'DISPLAY_NAME', value: "${params.DISPLAY_NAME} - ${image}")) + buildParams.add(string(name: 'BUILD_IMAGE_NAME', value: image)) + buildParams.add(string(name: 'SOURCE_AUTHOR', value: getGitAuthor())) + buildParams.add(string(name: 'SOURCE_BRANCH', value: getBuildBranch())) + + buildParams.add(string(name: 'MAVEN_ARTIFACTS_REPOSITORY', value: env.MAVEN_ARTIFACT_REPOSITORY)) + buildParams.add(string(name: 'BUILD_KOGITO_APPS_URI', value: params.APPS_URI)) + buildParams.add(string(name: 'BUILD_KOGITO_APPS_REF', value: params.APPS_REF)) + buildParams.add(booleanParam(name: 'SKIP_TESTS', value: params.SKIP_TESTS)) + buildParams.add(string(name: 'TESTS_KOGITO_EXAMPLES_URI', value: params.EXAMPLES_URI)) + buildParams.add(string(name: 'TESTS_KOGITO_EXAMPLES_REF', value: params.EXAMPLES_REF)) + + buildParams.add(booleanParam(name: 'DEPLOY_IMAGE', value: true)) + buildParams.add(booleanParam(name: 'DEPLOY_IMAGE_USE_OPENSHIFT_REGISTRY', value: isDeployImageInOpenshiftRegistry())) + buildParams.add(string(name: 'DEPLOY_IMAGE_REGISTRY_CREDENTIALS', value: getDeployImageRegistryCredentials())) + buildParams.add(string(name: 'DEPLOY_IMAGE_REGISTRY', value: getDeployImageRegistry())) + buildParams.add(string(name: 'DEPLOY_IMAGE_NAMESPACE', value: getDeployImageNamespace())) + buildParams.add(string(name: 'DEPLOY_IMAGE_NAME_SUFFIX', value: getDeployImageNameSuffix())) + buildParams.add(string(name: 'DEPLOY_IMAGE_TAG', value: getDeployImageTag())) + buildParams.add(booleanParam(name: 'DEPLOY_WITH_LATEST_TAG', value: isDeployLatestTag())) + + def job = build(job: 'kogito-images.build-image', wait: true, parameters: buildParams, propagate: false) + if (job.result != 'SUCCESS') { + if (job.result == 'UNSTABLE') { + registerTestFailedImage(image) + unstable("Tests on ${image} seems to have failed") + registerBuiltImage(image) + } else { + registerBuildFailedImage(image) + error("Error building ${image}. Please check the logs of the job: ${job.absoluteUrl}") + } + } else { + registerBuiltImage(image) + } + } + } +} + +void registerBuiltImage(String imageName) { + BUILT_IMAGES.add(imageName) +} + +void registerBuildFailedImage(String imageName) { + BUILD_FAILED_IMAGES.add(imageName) +} + +void registerTestFailedImage(String imageName) { + TEST_FAILED_IMAGES.add(imageName) +} + +List getBuiltImages() { + return BUILT_IMAGES +} + +List getBuildFailedImages() { + return BUILD_FAILED_IMAGES +} + +List getTestFailedImages() { + return TEST_FAILED_IMAGES +} + +//////////////////////////////////////////////////////////////////////// +// Deploy image information +//////////////////////////////////////////////////////////////////////// + +boolean isDeployImageInOpenshiftRegistry() { + return params.IMAGE_USE_OPENSHIFT_REGISTRY +} + +String getDeployImageRegistryCredentials() { + return params.IMAGE_REGISTRY_CREDENTIALS +} + +String getDeployImageRegistry() { + return isDeployImageInOpenshiftRegistry() ? getOpenShiftRegistry() : params.IMAGE_REGISTRY +} +String getDeployImageNamespace() { + return isDeployImageInOpenshiftRegistry() ? 'openshift' : params.IMAGE_NAMESPACE +} +String getDeployImageNameSuffix() { + return params.IMAGE_NAME_SUFFIX +} +String getDeployImageTag() { + if (params.IMAGE_TAG != '') { + return params.IMAGE_TAG + } else { + return sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim() + } +} + +boolean isDeployLatestTag() { + return params.DEPLOY_WITH_LATEST_TAG +} + +//////////////////////////////////////////////////////////////////////// +// utils +//////////////////////////////////////////////////////////////////////// + +String getRepoName() { + return env.REPO_NAME +} + +String getBuildBranch() { + return params.BUILD_BRANCH_NAME +} + +String getGitAuthor() { + return "${GIT_AUTHOR}" +} + +String getGitAuthorCredsId() { + return env.GIT_AUTHOR_CREDS_ID +} + +String getGitAuthorPushCredsId() { + return env.GIT_AUTHOR_PUSH_CREDS_ID +} + +String getPRBranch() { + return params.KOGITO_PR_BRANCH +} + +String getOpenShiftRegistry() { + String registry = '' + withCredentials([string(credentialsId: 'OPENSHIFT_REGISTRY', variable: 'OPENSHIFT_REGISTRY')]) { + registry = env.OPENSHIFT_REGISTRY + } + return registry +} + +void setDeployPropertyIfNeeded(String key, def value) { + if (value) { + deployProperties[key] = value + } +} + +String[] getImages() { + if (env.IMAGES_LIST) { + return env.IMAGES_LIST.split(',') + } + return sh(returnStdout: true, script: "make list | tr '\\n' ','").trim().split(',') +} + +String getQuarkusPlatformVersion() { + return params.QUARKUS_PLATFORM_VERSION +} + +String getProjectVersionDate() { + def projectVersionDate = (getCheckoutDatetime() =~ /(\d{4}-\d{2}-\d{2})/)[0][0] + return projectVersionDate.replace('-', '') +} + +String getProjectVersion(boolean keepSnapshotSuffix = true) { + def projectVersion = env.PROJECT_VERSION + if (keepSnapshotSuffix) { + return projectVersion.replace("-SNAPSHOT", "-${getProjectVersionDate()}-SNAPSHOT") + } + return projectVersion.replace("-SNAPSHOT", "-${getProjectVersionDate()}") +} diff --git a/.ci/jenkins/dsl/jobs.groovy b/.ci/jenkins/dsl/jobs.groovy index ea1c1c29..9205748f 100644 --- a/.ci/jenkins/dsl/jobs.groovy +++ b/.ci/jenkins/dsl/jobs.groovy @@ -321,3 +321,56 @@ void setupQuarkusUpdateJob() { "python scripts/update-repository.py --quarkus-platform-version %new_version%" ]) } + +void setupWeeklyDeployJob(JobType jobType) { + setupBuildImageJob(jobType) + + def jobParams = JobParamsUtils.getBasicJobParams(this, 'kogito-images-weekly-deploy', jobType, "${jenkins_path}/Jenkinsfile.weekly.deploy", 'Kogito Images Weekly Deploy') + JobParamsUtils.setupJobParamsAgentDockerBuilderImageConfiguration(this, jobParams) + jobParams.env.putAll([ + PROPERTIES_FILE_NAME: 'deployment.properties', + + MAX_REGISTRY_RETRIES: 3, + + JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", + + GIT_AUTHOR: "${GIT_AUTHOR_NAME}", + GIT_AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}", + GIT_AUTHOR_PUSH_CREDS_ID: "${GIT_AUTHOR_PUSH_CREDENTIALS_ID}", + + MAVEN_ARTIFACT_REPOSITORY: "${MAVEN_ARTIFACTS_REPOSITORY}", + DEFAULT_STAGING_REPOSITORY: "${MAVEN_NEXUS_STAGING_PROFILE_URL}", + + QUARKUS_PLATFORM_NEXUS_URL: Utils.getMavenQuarkusPlatformRepositoryUrl(this), + ]) + + jobParams.env.put('IMAGES_LIST', 'kogito-swf-devmode') + + KogitoJobTemplate.createPipelineJob(this, jobParams)?.with { + parameters { + stringParam('DISPLAY_NAME', '', 'Setup a specific build display name') + + stringParam('BUILD_BRANCH_NAME', "${GIT_BRANCH}", 'Set the Git branch to checkout') + + stringParam('APPS_URI', '', 'Git uri to the kogito-apps repository to use for tests.') + stringParam('APPS_REF', '', 'Git reference (branch/tag) to the kogito-apps repository to use for building. Default to BUILD_BRANCH_NAME.') + + // Build&Test information + booleanParam('SKIP_TESTS', false, 'Skip tests') + stringParam('EXAMPLES_URI', '', 'Git uri to the kogito-examples repository to use for tests.') + stringParam('EXAMPLES_REF', '', 'Git reference (branch/tag) to the kogito-examples repository to use for tests.') + + // Deploy information + booleanParam('IMAGE_USE_OPENSHIFT_REGISTRY', false, 'Set to true if image should be deployed in Openshift registry.In this case, IMAGE_REGISTRY_CREDENTIALS, IMAGE_REGISTRY and IMAGE_NAMESPACE parameters will be ignored') + stringParam('IMAGE_REGISTRY_CREDENTIALS', "${CLOUD_IMAGE_REGISTRY_CREDENTIALS}", 'Image registry credentials to use to deploy images. Will be ignored if no IMAGE_REGISTRY is given') + stringParam('IMAGE_REGISTRY', "${CLOUD_IMAGE_REGISTRY}", 'Image registry to use to deploy images') + stringParam('IMAGE_NAMESPACE', "${CLOUD_IMAGE_NAMESPACE}", 'Image namespace to use to deploy images') + stringParam('IMAGE_NAME_SUFFIX', '', 'Image name suffix to use to deploy images. In case you need to change the final image name, you can add a suffix to it.') + stringParam('IMAGE_TAG', '', 'Image tag to use to deploy images') + booleanParam('DEPLOY_WITH_LATEST_TAG', false, 'Set to true if you want the deployed images to also be with the `latest` tag') + + stringParam('KOGITO_PR_BRANCH', '', 'PR branch name') + booleanParam('SEND_NOTIFICATION', false, 'In case you want the pipeline to send a notification on CI channel for this run.') + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
