This is an automated email from the ASF dual-hosted git repository.
rantunes pushed a commit to branch main
in repository
https://gitbox.apache.org/repos/asf/incubator-kie-kogito-images.git
The following commit(s) were added to refs/heads/main by this push:
new c223f0ef kie-issues#821: Kogito-images weekly job (#1740)
c223f0ef is described below
commit c223f0ef0a560c8b3f1452ec5d0271afe2891127
Author: Rodrigo Antunes <[email protected]>
AuthorDate: Thu Feb 8 08:14:50 2024 -0300
kie-issues#821: Kogito-images weekly job (#1740)
* Kogito Images weekly job
* Add kogito-swf-builder image and weekly-latest tag
---
.ci/jenkins/Jenkinsfile.build-image | 10 +-
.ci/jenkins/Jenkinsfile.weekly.deploy | 347 ++++++++++++++++++++++++++++++++++
.ci/jenkins/dsl/jobs.groovy | 56 ++++++
scripts/manage-kogito-version.py | 6 +-
4 files changed, 415 insertions(+), 4 deletions(-)
diff --git a/.ci/jenkins/Jenkinsfile.build-image
b/.ci/jenkins/Jenkinsfile.build-image
index 2a51da90..75347f85 100644
--- a/.ci/jenkins/Jenkinsfile.build-image
+++ b/.ci/jenkins/Jenkinsfile.build-image
@@ -151,7 +151,8 @@ pipeline {
int retries = Integer.parseInt(env.MAX_REGISTRY_RETRIES)
String imageTag = getBuiltImageTag()
if (isDeployLatestTag()) {
- cloud.skopeoCopyRegistryImages(imageTag,
getBuiltImageTag('latest'), retries)
+ latestTag =
getTriggeringProjectName().contains('weekly') ? 'weekly-latest' : 'latest'
+ cloud.skopeoCopyRegistryImages(imageTag,
getBuiltImageTag(latestTag), retries)
}
try {
String reducedTag =
cloud.getReducedTag(getDeployImageTag())
@@ -407,3 +408,10 @@ Map getRepositoryInfo() {
credentials_id: getGitAuthorCredentialsId(),
]
}
+
+String getTriggeringProjectName() {
+ if (currentBuild.upstreamBuilds) {
+ return currentBuild.upstreamBuilds[0].projectName
+ }
+ return ""
+}
diff --git a/.ci/jenkins/Jenkinsfile.weekly.deploy
b/.ci/jenkins/Jenkinsfile.weekly.deploy
new file mode 100644
index 00000000..045e2366
--- /dev/null
+++ b/.ci/jenkins/Jenkinsfile.weekly.deploy
@@ -0,0 +1,347 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@Library('jenkins-pipeline-shared-libraries')_
+
+deployProperties = [:]
+
+changesDone = false
+
+BUILT_IMAGES = Collections.synchronizedList([])
+BUILD_FAILED_IMAGES = Collections.synchronizedList([])
+TEST_FAILED_IMAGES = Collections.synchronizedList([])
+
+pipeline {
+ agent {
+ docker {
+ image env.AGENT_DOCKER_BUILDER_IMAGE
+ args env.AGENT_DOCKER_BUILDER_ARGS
+ }
+ }
+
+ options {
+ timeout(time: 240, unit: 'MINUTES')
+ skipDefaultCheckout()
+ }
+
+ environment {
+ KOGITO_CI_EMAIL_TO = credentials("${JENKINS_EMAIL_CREDS_ID}")
+ }
+
+ stages {
+ stage('Initialization') {
+ steps {
+ script {
+ util.waitForDocker()
+ currentBuild.displayName = params.DISPLAY_NAME ?:
currentBuild.displayName
+
+ dir(getRepoName()) {
+ checkoutRepo()
+ env.PROJECT_VERSION = sh(returnStdout: true, script:
"python scripts/retrieve_version.py").trim()
+ }
+ }
+ }
+ post {
+ always {
+ script {
+ setDeployPropertyIfNeeded('git.branch',
getBuildBranch())
+ setDeployPropertyIfNeeded('git.author', getGitAuthor())
+ setDeployPropertyIfNeeded('project.version',
getProjectVersion())
+ }
+ }
+ }
+ }
+
+ stage('Update project version') {
+ steps {
+ script {
+ dir(getRepoName()) {
+ versionCmd = "python scripts/manage-kogito-version.py
--bump-to ${getProjectVersion()} --confirm"
+ if (getBuildBranch() != 'main') {
+ versionCmd += " --examples-ref
'nightly-'${getBuildBranch()}"
+ }
+ sh versionCmd
+ }
+ }
+ }
+ }
+
+ stage('Validate CeKit Image and Modules descriptors') {
+ steps {
+ script {
+ dir(getRepoName()) {
+ sh '''
+ curl -Ls
https://github.com/kiegroup/kie-cloud-tools/releases/download/v1.3.6/cekit-image-validator-runner.tgz
--output cekit-image-validator-runner.tgz
+ tar -xzvf cekit-image-validator-runner.tgz
+ chmod +x cekit-image-validator-runner
+ '''
+ sh './cekit-image-validator-runner modules/'
+ getImages().each { image -> sh
"./cekit-image-validator-runner ${image}-image.yaml" }
+ }
+ }
+ }
+ }
+
+ stage('Build, Push & Test Images') {
+ steps {
+ script {
+ dir(getRepoName()) {
+ parallelStages = [:]
+ getImages().each { image ->
+ parallelStages["Build&Test ${image}"] =
createBuildAndTestStageClosure(image)
+ }
+ parallel parallelStages
+ }
+ }
+ }
+ post {
+ always {
+ script {
+ // Store image deployment information
+ String imgPrefix = "${getRepoName()}.image"
+ setDeployPropertyIfNeeded("${imgPrefix}.registry",
getDeployImageRegistry())
+ setDeployPropertyIfNeeded("${imgPrefix}.namespace",
getDeployImageNamespace())
+ setDeployPropertyIfNeeded("${imgPrefix}.names",
getBuiltImages().join(','))
+ setDeployPropertyIfNeeded("${imgPrefix}.name-suffix",
getDeployImageNameSuffix())
+ setDeployPropertyIfNeeded("${imgPrefix}.tag",
getDeployImageTag())
+ }
+ }
+ }
+ }
+
+ stage('Create and push a new tag') {
+ steps {
+ script {
+ projectVersion = getProjectVersion(false)
+ dir(getRepoName()) {
+
githubscm.setUserConfigFromCreds(getGitAuthorPushCredsId())
+ githubscm.tagRepository(projectVersion)
+ githubscm.pushRemoteTag('origin', projectVersion,
getGitAuthorPushCredsId())
+ }
+ }
+ }
+ }
+
+ stage('Finalize') {
+ steps {
+ script {
+ if (getBuildFailedImages()) {
+ currentBuild.currentResult = 'FAILURE'
+ }
+ }
+ }
+ }
+ }
+ post {
+ always {
+ script {
+ def propertiesStr = deployProperties.collect { entry ->
"${entry.key}=${entry.value}" }.join('\n')
+ writeFile(file : env.PROPERTIES_FILE_NAME , text :
propertiesStr)
+ archiveArtifacts artifacts: env.PROPERTIES_FILE_NAME,
allowEmptyArchive:true
+ }
+ }
+ cleanup {
+ cleanWs()
+ }
+ unsuccessful {
+ sendUnsuccessfulNotification()
+ }
+ }
+}
+
+void sendUnsuccessfulNotification() {
+ if (params.SEND_NOTIFICATION) {
+ mailer.sendMarkdownTestSummaryNotification('Deploy',
getNotificationSubject(), [env.KOGITO_CI_EMAIL_TO])
+ } else {
+ echo 'No notification sent per configuration'
+ }
+}
+
+void sendNotification(String body) {
+ emailext body: body,
+ subject: getNotificationSubject(),
+ to: env.KOGITO_CI_EMAIL_TO
+}
+
+String getNotificationSubject() {
+ return "[${getBuildBranch()}] Kogito Images"
+}
+
+void checkoutRepo() {
+ deleteDir()
+ checkout(githubscm.resolveRepository(getRepoName(), getGitAuthor(),
getBuildBranch(), false, getGitAuthorCredsId()))
+}
+
+void createBuildAndTestStageClosure(String image) {
+ return {
+ stage("Build&Test ${image}") {
+ List buildParams = []
+ buildParams.add(string(name: 'DISPLAY_NAME', value:
"${params.DISPLAY_NAME} - ${image}"))
+ buildParams.add(string(name: 'BUILD_IMAGE_NAME', value: image))
+ buildParams.add(string(name: 'SOURCE_AUTHOR', value:
getGitAuthor()))
+ buildParams.add(string(name: 'SOURCE_BRANCH', value:
getBuildBranch()))
+
+ buildParams.add(string(name: 'MAVEN_ARTIFACTS_REPOSITORY', value:
env.MAVEN_ARTIFACT_REPOSITORY))
+ buildParams.add(string(name: 'BUILD_KOGITO_APPS_URI', value:
params.APPS_URI))
+ buildParams.add(string(name: 'BUILD_KOGITO_APPS_REF', value:
params.APPS_REF))
+ buildParams.add(booleanParam(name: 'SKIP_TESTS', value:
params.SKIP_TESTS))
+ buildParams.add(string(name: 'TESTS_KOGITO_EXAMPLES_URI', value:
params.EXAMPLES_URI))
+ buildParams.add(string(name: 'TESTS_KOGITO_EXAMPLES_REF', value:
params.EXAMPLES_REF))
+
+ buildParams.add(booleanParam(name: 'DEPLOY_IMAGE', value: true))
+ buildParams.add(booleanParam(name:
'DEPLOY_IMAGE_USE_OPENSHIFT_REGISTRY', value:
isDeployImageInOpenshiftRegistry()))
+ buildParams.add(string(name: 'DEPLOY_IMAGE_REGISTRY_CREDENTIALS',
value: getDeployImageRegistryCredentials()))
+ buildParams.add(string(name: 'DEPLOY_IMAGE_REGISTRY', value:
getDeployImageRegistry()))
+ buildParams.add(string(name: 'DEPLOY_IMAGE_NAMESPACE', value:
getDeployImageNamespace()))
+ buildParams.add(string(name: 'DEPLOY_IMAGE_NAME_SUFFIX', value:
getDeployImageNameSuffix()))
+ buildParams.add(string(name: 'DEPLOY_IMAGE_TAG', value:
getDeployImageTag()))
+ buildParams.add(booleanParam(name: 'DEPLOY_WITH_LATEST_TAG',
value: isDeployLatestTag()))
+
+ def job = build(job: 'kogito-images.build-image', wait: true,
parameters: buildParams, propagate: false)
+ if (job.result != 'SUCCESS') {
+ if (job.result == 'UNSTABLE') {
+ registerTestFailedImage(image)
+ unstable("Tests on ${image} seems to have failed")
+ registerBuiltImage(image)
+ } else {
+ registerBuildFailedImage(image)
+ error("Error building ${image}. Please check the logs of
the job: ${job.absoluteUrl}")
+ }
+ } else {
+ registerBuiltImage(image)
+ }
+ }
+ }
+}
+
+void registerBuiltImage(String imageName) {
+ BUILT_IMAGES.add(imageName)
+}
+
+void registerBuildFailedImage(String imageName) {
+ BUILD_FAILED_IMAGES.add(imageName)
+}
+
+void registerTestFailedImage(String imageName) {
+ TEST_FAILED_IMAGES.add(imageName)
+}
+
+List getBuiltImages() {
+ return BUILT_IMAGES
+}
+
+List getBuildFailedImages() {
+ return BUILD_FAILED_IMAGES
+}
+
+List getTestFailedImages() {
+ return TEST_FAILED_IMAGES
+}
+
+////////////////////////////////////////////////////////////////////////
+// Deploy image information
+////////////////////////////////////////////////////////////////////////
+
+boolean isDeployImageInOpenshiftRegistry() {
+ return params.IMAGE_USE_OPENSHIFT_REGISTRY
+}
+
+String getDeployImageRegistryCredentials() {
+ return params.IMAGE_REGISTRY_CREDENTIALS
+}
+
+String getDeployImageRegistry() {
+ return isDeployImageInOpenshiftRegistry() ? getOpenShiftRegistry() :
params.IMAGE_REGISTRY
+}
+String getDeployImageNamespace() {
+ return isDeployImageInOpenshiftRegistry() ? 'openshift' :
params.IMAGE_NAMESPACE
+}
+String getDeployImageNameSuffix() {
+ return params.IMAGE_NAME_SUFFIX
+}
+String getDeployImageTag() {
+ return getProjectVersion(false)
+}
+
+boolean isDeployLatestTag() {
+ return params.DEPLOY_WITH_LATEST_TAG
+}
+
+////////////////////////////////////////////////////////////////////////
+// utils
+////////////////////////////////////////////////////////////////////////
+
+String getRepoName() {
+ return env.REPO_NAME
+}
+
+String getBuildBranch() {
+ return params.BUILD_BRANCH_NAME
+}
+
+String getGitAuthor() {
+ return "${GIT_AUTHOR}"
+}
+
+String getGitAuthorCredsId() {
+ return env.GIT_AUTHOR_CREDS_ID
+}
+
+String getGitAuthorPushCredsId() {
+ return env.GIT_AUTHOR_PUSH_CREDS_ID
+}
+
+String getOpenShiftRegistry() {
+ String registry = ''
+ withCredentials([string(credentialsId: 'OPENSHIFT_REGISTRY', variable:
'OPENSHIFT_REGISTRY')]) {
+ registry = env.OPENSHIFT_REGISTRY
+ }
+ return registry
+}
+
+void setDeployPropertyIfNeeded(String key, def value) {
+ if (value) {
+ deployProperties[key] = value
+ }
+}
+
+String[] getImages() {
+ if (env.IMAGES_LIST) {
+ return env.IMAGES_LIST.split(',')
+ }
+ return sh(returnStdout: true, script: "make list | tr '\\n'
','").trim().split(',')
+}
+
+String getQuarkusPlatformVersion() {
+ return params.QUARKUS_PLATFORM_VERSION
+}
+
+String getCheckoutDatetime() {
+ return params.GIT_CHECKOUT_DATETIME
+}
+
+String getProjectVersionDate() {
+ def projectVersionDate = (getCheckoutDatetime() =~
/(\d{4}-\d{2}-\d{2})/)[0][0]
+ return projectVersionDate.replace('-', '')
+}
+
+String getProjectVersion(boolean keepSnapshotSuffix = true) {
+ def projectVersion = env.PROJECT_VERSION
+ if (keepSnapshotSuffix) {
+ return projectVersion.replace("-SNAPSHOT",
"-${getProjectVersionDate()}-SNAPSHOT")
+ }
+ return projectVersion.replace("-SNAPSHOT", "-${getProjectVersionDate()}")
+}
diff --git a/.ci/jenkins/dsl/jobs.groovy b/.ci/jenkins/dsl/jobs.groovy
index ea1c1c29..5ed082bb 100644
--- a/.ci/jenkins/dsl/jobs.groovy
+++ b/.ci/jenkins/dsl/jobs.groovy
@@ -45,6 +45,9 @@ createSetupBranchJob()
setupDeployJob(JobType.NIGHTLY)
KogitoJobUtils.createEnvironmentIntegrationBranchNightlyJob(this,
'quarkus-lts')
+// Weekly jobs
+setupWeeklyDeployJob(JobType.OTHER)
+
// Release jobs
setupDeployJob(JobType.RELEASE)
setupPromoteJob(JobType.RELEASE)
@@ -321,3 +324,56 @@ void setupQuarkusUpdateJob() {
"python scripts/update-repository.py --quarkus-platform-version
%new_version%"
])
}
+
+void setupWeeklyDeployJob(JobType jobType) {
+ setupBuildImageJob(jobType)
+
+ def jobParams = JobParamsUtils.getBasicJobParams(this,
'kogito-images.weekly-deploy', jobType,
"${jenkins_path}/Jenkinsfile.weekly.deploy", 'Kogito Images Weekly Deploy')
+ JobParamsUtils.setupJobParamsAgentDockerBuilderImageConfiguration(this,
jobParams)
+ jobParams.env.putAll([
+ PROPERTIES_FILE_NAME: 'deployment.properties',
+
+ MAX_REGISTRY_RETRIES: 3,
+
+ JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}",
+
+ GIT_AUTHOR: "${GIT_AUTHOR_NAME}",
+ GIT_AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}",
+ GIT_AUTHOR_PUSH_CREDS_ID: "${GIT_AUTHOR_PUSH_CREDENTIALS_ID}",
+
+ MAVEN_ARTIFACT_REPOSITORY: "${MAVEN_ARTIFACTS_REPOSITORY}",
+ DEFAULT_STAGING_REPOSITORY: "${MAVEN_NEXUS_STAGING_PROFILE_URL}",
+
+ QUARKUS_PLATFORM_NEXUS_URL:
Utils.getMavenQuarkusPlatformRepositoryUrl(this),
+ ])
+
+ jobParams.env.put('IMAGES_LIST', 'kogito-swf-builder,kogito-swf-devmode')
+
+ KogitoJobTemplate.createPipelineJob(this, jobParams)?.with {
+ parameters {
+ stringParam('DISPLAY_NAME', '', 'Setup a specific build display
name')
+
+ stringParam('BUILD_BRANCH_NAME', "${GIT_BRANCH}", 'Set the Git
branch to checkout')
+
+ stringParam('APPS_URI', '', 'Git uri to the kogito-apps repository
to use for tests.')
+ stringParam('APPS_REF', '', 'Git reference (branch/tag) to the
kogito-apps repository to use for building. Default to BUILD_BRANCH_NAME.')
+
+ // Build&Test information
+ booleanParam('SKIP_TESTS', false, 'Skip tests')
+ stringParam('EXAMPLES_URI', '', 'Git uri to the kogito-examples
repository to use for tests.')
+ stringParam('EXAMPLES_REF', '', 'Git reference (branch/tag) to the
kogito-examples repository to use for tests.')
+
+ // Deploy information
+ booleanParam('IMAGE_USE_OPENSHIFT_REGISTRY', false, 'Set to true
if image should be deployed in Openshift registry.In this case,
IMAGE_REGISTRY_CREDENTIALS, IMAGE_REGISTRY and IMAGE_NAMESPACE parameters will
be ignored')
+ stringParam('IMAGE_REGISTRY_CREDENTIALS',
"${CLOUD_IMAGE_REGISTRY_CREDENTIALS}", 'Image registry credentials to use to
deploy images. Will be ignored if no IMAGE_REGISTRY is given')
+ stringParam('IMAGE_REGISTRY', "${CLOUD_IMAGE_REGISTRY}", 'Image
registry to use to deploy images')
+ stringParam('IMAGE_NAMESPACE', "${CLOUD_IMAGE_NAMESPACE}", 'Image
namespace to use to deploy images')
+ stringParam('IMAGE_NAME_SUFFIX', '', 'Image name suffix to use to
deploy images. In case you need to change the final image name, you can add a
suffix to it.')
+ booleanParam('DEPLOY_WITH_LATEST_TAG', false, 'Set to true if you
want the deployed images to also be with the `latest` tag')
+
+ stringParam('GIT_CHECKOUT_DATETIME', '', 'Git checkout date and
time - (Y-m-d H:i)')
+
+ booleanParam('SEND_NOTIFICATION', false, 'In case you want the
pipeline to send a notification on CI channel for this run.')
+ }
+ }
+}
diff --git a/scripts/manage-kogito-version.py b/scripts/manage-kogito-version.py
index 04cb161f..10950d22 100644
--- a/scripts/manage-kogito-version.py
+++ b/scripts/manage-kogito-version.py
@@ -51,9 +51,9 @@ if __name__ == "__main__":
if args.bump_to:
# validate if the provided version is valid.
- # e.g. 1.10.0 or 1.0.0-rc1
- pattern = '\d+.\d+.(\d+$|\d+-rc\d+$|\d+-snapshot$)'
- regex = re.compile(r'\d+.\d+.(\d+$|\d+-rc\d+|\d+-snapshot$)')
+ # e.g. 1.10.0, 1.0.0-rc1, 999-snapshot or 999-20240101-snapshot
+ pattern = r'(\d+.\d+.)?(\d+$|\d+-rc\d+$|\d+(-\d{8})?-snapshot$)'
+ regex = re.compile(pattern, re.IGNORECASE)
valid = regex.match(args.bump_to)
examples_ref = ""
if valid:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
