This is an automated email from the ASF dual-hosted git repository. ricardozanini pushed a commit to branch radtriste-fix-pipelines in repository https://gitbox.apache.org/repos/asf/incubator-kie-kogito-images.git
commit 4e602316a400f55cee2a0428b06a742c1030bcad Author: radtriste <[email protected]> AuthorDate: Tue Oct 17 10:43:52 2023 +0200 Fix pipelines --- .ci/jenkins/Jenkinsfile.build-image | 8 +- .ci/jenkins/Jenkinsfile.deploy | 160 ++++++++++++++++++----------------- .ci/jenkins/Jenkinsfile.setup-branch | 2 +- .ci/jenkins/dsl/jobs.groovy | 2 +- Makefile | 6 +- scripts/README.md | 4 +- scripts/common.py | 2 +- scripts/list-images.py | 2 +- scripts/manage-kogito-version.py | 2 +- scripts/push-local-registry.sh | 2 +- scripts/push-staging.py | 2 +- scripts/retrieve_version.py | 4 +- scripts/update-repository.py | 4 +- tests/test-apps/clone-repo.sh | 2 +- 14 files changed, 102 insertions(+), 100 deletions(-) diff --git a/.ci/jenkins/Jenkinsfile.build-image b/.ci/jenkins/Jenkinsfile.build-image index f13a6404..81889abd 100644 --- a/.ci/jenkins/Jenkinsfile.build-image +++ b/.ci/jenkins/Jenkinsfile.build-image @@ -67,12 +67,12 @@ pipeline { env.MAVEN_MIRROR_URL = env.MAVEN_MIRROR_REPOSITORY // Ignore self-signed certificates if MAVEN_MIRROR_URL is defined - runPythonCommand("python3 scripts/update-repository.py --build-maven-mirror-url ${MAVEN_MIRROR_URL} --ignore-self-signed-cert") + runPythonCommand("python scripts/update-repository.py --build-maven-mirror-url ${MAVEN_MIRROR_URL} --ignore-self-signed-cert") } if (getMavenArtifactRepository()) { echo "Setup Repository url to '${getMavenArtifactRepository()}'" - runPythonCommand("python3 scripts/update-repository.py --repo-url ${getMavenArtifactRepository()}") + runPythonCommand("python scripts/update-repository.py --repo-url ${getMavenArtifactRepository()}") } } } @@ -89,7 +89,7 @@ pipeline { // echo "[WARN] Artifacts repository defined in env will override the quarkus platform URL in tests. Make sure the platform artifacts are available on that artifacts repository (you can use a maven group)" // } // // Setup quarkus platform repo configuration - // runPythonCommand("python3 scripts/update-repository.py --repo-url ${getQuarkusPlatformURL()} --ignore-self-signed-cert --quarkus-platform-version ${quarkusPlatformVersion}") + // runPythonCommand("python scripts/update-repository.py --repo-url ${getQuarkusPlatformURL()} --ignore-self-signed-cert --quarkus-platform-version ${quarkusPlatformVersion}") // } // } // } @@ -153,7 +153,7 @@ pipeline { } steps { script { - updateTestsCommand = 'python3 scripts/update-repository.py --tests-only' + updateTestsCommand = 'python scripts/update-repository.py --tests-only' updateTestsCommand += getMavenArtifactRepository() ? " --repo-url ${getMavenArtifactRepository()}" : '' updateTestsCommand += getTestsKogitoExamplesRef() ? " --examples-ref ${getTestsKogitoExamplesRef()}" : '' updateTestsCommand += getTestsKogitoExamplesURI() ? " --examples-uri ${getTestsKogitoExamplesURI()}" : '' diff --git a/.ci/jenkins/Jenkinsfile.deploy b/.ci/jenkins/Jenkinsfile.deploy index 83697af7..00037337 100644 --- a/.ci/jenkins/Jenkinsfile.deploy +++ b/.ci/jenkins/Jenkinsfile.deploy @@ -14,27 +14,25 @@ pipeline { image env.AGENT_DOCKER_BUILDER_IMAGE args env.AGENT_DOCKER_BUILDER_ARGS } - } + } options { - timeout(time: 120, unit: 'MINUTES') + timeout(time: 240, unit: 'MINUTES') } environment { KOGITO_CI_EMAIL_TO = credentials("${JENKINS_EMAIL_CREDS_ID}") - - OPENSHIFT_REGISTRY = credentials('OPENSHIFT_REGISTRY') } stages { stage('Initialization') { steps { script { - clean() - currentBuild.displayName = params.DISPLAY_NAME ?: currentBuild.displayName - checkoutRepo() + dir(getRepoName()) { + checkoutRepo() + } if (isRelease()) { // Verify version is set and if on right release branch @@ -62,10 +60,12 @@ pipeline { } steps { script { - if (githubscm.isBranchExist('origin',getPRBranch())) { - githubscm.removeRemoteBranch('origin', getPRBranch()) + dir(getRepoName()) { + if (githubscm.isBranchExist('origin',getPRBranch())) { + githubscm.removeRemoteBranch('origin', getPRBranch()) + } + githubscm.createBranch(getPRBranch()) } - githubscm.createBranch(getPRBranch()) } } } @@ -76,12 +76,14 @@ pipeline { } steps { script { - if (getQuarkusPlatformVersion()) { - runPythonCommand("python3 scripts/update-repository.py --quarkus-platform-version ${getQuarkusPlatformVersion()}") + dir(getRepoName()) { + if (getQuarkusPlatformVersion()) { + sh "python scripts/update-repository.py --quarkus-platform-version ${getQuarkusPlatformVersion()}" - commitAndPushChanges("Update Quarkus Platform version to ${getQuarkusPlatformVersion()}") - } else { - echo 'No new quarkus version given for the release. Statu quo ...' + commitAndPushChanges("Update Quarkus Platform version to ${getQuarkusPlatformVersion()}") + } else { + echo 'No new quarkus version given for the release. Statu quo ...' + } } } } @@ -93,16 +95,18 @@ pipeline { } steps { script { - versionCmd = "python3 scripts/manage-kogito-version.py --bump-to ${getProjectVersion()} --confirm" - if (getBuildBranch() != 'main') { - versionCmd += " --examples-ref ${!isRelease() ? 'nightly-' : ''}${getBuildBranch()}" - } - if (getKogitoArtifactsVersion()) { - versionCmd += " --artifacts-version ${getKogitoArtifactsVersion()}" - } - runPythonCommand(versionCmd) + dir(getRepoName()) { + versionCmd = "python scripts/manage-kogito-version.py --bump-to ${getProjectVersion()} --confirm" + if (getBuildBranch() != 'main') { + versionCmd += " --examples-ref ${!isRelease() ? 'nightly-' : ''}${getBuildBranch()}" + } + if (getKogitoArtifactsVersion()) { + versionCmd += " --artifacts-version ${getKogitoArtifactsVersion()}" + } + sh versionCmd - commitAndPushChanges("Update project version to ${getProjectVersion()}") + commitAndPushChanges("Update project version to ${getProjectVersion()}") + } } } } @@ -110,18 +114,15 @@ pipeline { stage('Validate CeKit Image and Modules descriptors') { steps { script { - sh ''' - curl -Ls https://github.com/kiegroup/kie-cloud-tools/releases/download/v1.3.6/cekit-image-validator-runner.tgz --output cekit-image-validator-runner.tgz - tar -xzvf cekit-image-validator-runner.tgz - chmod +x cekit-image-validator-runner - ''' - sh './cekit-image-validator-runner modules/' - getImages().each { image -> sh "./cekit-image-validator-runner ${image}-image.yaml" } - } - } - post { - always { - sh 'rm -rf cekit-image-validator-runner*' + dir(getRepoName()) { + sh ''' + curl -Ls https://github.com/kiegroup/kie-cloud-tools/releases/download/v1.3.6/cekit-image-validator-runner.tgz --output cekit-image-validator-runner.tgz + tar -xzvf cekit-image-validator-runner.tgz + chmod +x cekit-image-validator-runner + ''' + sh './cekit-image-validator-runner modules/' + getImages().each { image -> sh "./cekit-image-validator-runner ${image}-image.yaml" } + } } } } @@ -129,11 +130,13 @@ pipeline { stage('Build, Push & Test Images') { steps { script { - parallelStages = [:] - getImages().each { image -> - parallelStages["Build&Test ${image}"] = createBuildAndTestStageClosure(image) + dir(getRepoName()) { + parallelStages = [:] + getImages().each { image -> + parallelStages["Build&Test ${image}"] = createBuildAndTestStageClosure(image) + } + parallel parallelStages } - parallel parallelStages } } post { @@ -159,37 +162,39 @@ pipeline { } steps { script { - def commitMsg = "[${getBuildBranch()}] Update Maven artifacts" - def prBody = "Generated by build ${BUILD_TAG}: ${BUILD_URL}." - if (isRelease()) { - commitMsg = "[${getBuildBranch()}] Update project version to ${getProjectVersion()}" - prBody += '\nPlease do not merge, it should be merged automatically.' - } else if (currentBuild.currentResult != 'SUCCESS') { - commitMsg += " (${currentBuild.currentResult})" - prBody += '\n\nSuccessful images:\n' - getBuiltImages().each { - prBody += "- ${it}\n" - } - if (getBuildFailedImages()) { - prBody += '\nBuild failures on those images:\n' - getBuildFailedImages().each { + dir(getRepoName()) { + def commitMsg = "[${getBuildBranch()}] Update Maven artifacts" + def prBody = "Generated by build ${BUILD_TAG}: ${BUILD_URL}." + if (isRelease()) { + commitMsg = "[${getBuildBranch()}] Update project version to ${getProjectVersion()}" + prBody += '\nPlease do not merge, it should be merged automatically.' + } else if (currentBuild.currentResult != 'SUCCESS') { + commitMsg += " (${currentBuild.currentResult})" + prBody += '\n\nSuccessful images:\n' + getBuiltImages().each { prBody += "- ${it}\n" } - } else { - prBody += '\nImages were all successfully built but some other problem occured in the pipeline execution...\n' - } - if (getTestFailedImages()) { - prBody += '\nTest failures on those images:\n' - getTestFailedImages().each { - prBody += "- ${it}\n" + if (getBuildFailedImages()) { + prBody += '\nBuild failures on those images:\n' + getBuildFailedImages().each { + prBody += "- ${it}\n" + } + } else { + prBody += '\nImages were all successfully built but some other problem occured in the pipeline execution...\n' } - } else { - prBody += '\nImages were all successfully built but some other problem occured in the pipeline execution...\n' + if (getTestFailedImages()) { + prBody += '\nTest failures on those images:\n' + getTestFailedImages().each { + prBody += "- ${it}\n" + } + } else { + prBody += '\nImages were all successfully built but some other problem occured in the pipeline execution...\n' + } + prBody += '\nSee build url above for more information' } - prBody += '\nSee build url above for more information' + String prLink = githubscm.createPR(commitMsg, prBody, getBuildBranch(), getGitAuthorCredsID()) + deployProperties["${getRepoName()}.pr.link"] = prLink } - String prLink = githubscm.createPR(commitMsg, prBody, getBuildBranch(), getGitAuthorCredsID()) - deployProperties["${getRepoName()}.pr.link"] = prLink } } post { @@ -224,9 +229,6 @@ pipeline { unsuccessful { sendUnsuccessfulNotification() } - cleanup { - clean() - } } } @@ -259,10 +261,6 @@ void commitAndPushChanges(String commitMsg) { changesDone = true } -void clean() { - util.cleanNode() -} - void createBuildAndTestStageClosure(String image) { return { stage("Build&Test ${image}") { @@ -348,7 +346,7 @@ String getDeployImageRegistryCredentials() { } String getDeployImageRegistry() { - return isDeployImageInOpenshiftRegistry() ? env.OPENSHIFT_REGISTRY : params.IMAGE_REGISTRY + return isDeployImageInOpenshiftRegistry() ? getOpenShiftRegistry() : params.IMAGE_REGISTRY } String getDeployImageNamespace() { return isDeployImageInOpenshiftRegistry() ? 'openshift' : params.IMAGE_NAMESPACE @@ -404,6 +402,14 @@ String getKogitoArtifactsVersion() { return params.KOGITO_ARTIFACTS_VERSION } +String getOpenShiftRegistry() { + String registry = '' + withCredentials([string(credentialsId: 'OPENSHIFT_REGISTRY', variable: 'OPENSHIFT_REGISTRY')]) { + registry = env.OPENSHIFT_REGISTRY + } + return registry +} + void setDeployPropertyIfNeeded(String key, def value) { if (value) { deployProperties[key] = value @@ -414,11 +420,7 @@ String[] getImages() { if (env.IMAGES_LIST) { return env.IMAGES_LIST.split(',') } - return runPythonCommand("make list | tr '\\n' ','", true).trim().split(',') -} - -void runPythonCommand(String cmd, boolean stdout = false) { - return sh(returnStdout: stdout, script: cmd) + return sh(returnStdout: true, script: "make list | tr '\\n' ','").trim().split(',') } String getQuarkusPlatformVersion() { diff --git a/.ci/jenkins/Jenkinsfile.setup-branch b/.ci/jenkins/Jenkinsfile.setup-branch index 182689cb..17c2ffb7 100644 --- a/.ci/jenkins/Jenkinsfile.setup-branch +++ b/.ci/jenkins/Jenkinsfile.setup-branch @@ -35,7 +35,7 @@ pipeline { steps { script { dir(getRepoName()) { - versionCmd = "python3 scripts/manage-kogito-version.py --bump-to ${getKogitoVersion()} --confirm" + versionCmd = "python scripts/manage-kogito-version.py --bump-to ${getKogitoVersion()} --confirm" versionCmd += " --examples-ref nightly-${getBuildBranch()}" if (getKogitoArtifactsVersion()) { versionCmd += " --artifacts-version ${getKogitoArtifactsVersion()}" diff --git a/.ci/jenkins/dsl/jobs.groovy b/.ci/jenkins/dsl/jobs.groovy index fb18ed05..8e533131 100644 --- a/.ci/jenkins/dsl/jobs.groovy +++ b/.ci/jenkins/dsl/jobs.groovy @@ -256,6 +256,6 @@ void setupPromoteJob(JobType jobType) { void setupQuarkusUpdateJob() { KogitoJobUtils.createQuarkusUpdateToolsJob(this, 'kogito-images', [:], [:], [], [ - "source ~/virtenvs/cekit/bin/activate && python3 scripts/update-repository.py --quarkus-platform-version %new_version%" + "python scripts/update-repository.py --quarkus-platform-version %new_version%" ]) } diff --git a/Makefile b/Makefile index 4e2b4657..150a74a6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -IMAGE_VERSION := $(shell python3 scripts/retrieve_version.py) +IMAGE_VERSION := $(shell python scripts/retrieve_version.py) SHORTENED_LATEST_VERSION := $(shell echo $(IMAGE_VERSION) | awk -F. '{print $$1"."$$2}') KOGITO_APPS_TARGET_BRANCH ?= main KOGITO_APPS_TARGET_URI ?= https://github.com/kiegroup/kogito-apps.git @@ -19,7 +19,7 @@ endif .PHONY: list list: - @python3 scripts/list-images.py $(arg) + @python scripts/list-images.py $(arg) .PHONY: display-image-version display-image-version: @@ -99,7 +99,7 @@ endif .PHONY: push-staging push-staging: build _push-staging _push-staging: - python3 scripts/push-staging.py ${override} + python scripts/push-staging.py ${override} # push to local registry, useful to push the built images to local registry diff --git a/scripts/README.md b/scripts/README.md index b11e68cf..6b13cceb 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -50,13 +50,13 @@ Utilitary script used to retrieve all images that can be built on this repo, the the community image list: ```bash -$ python3 list-images.py +$ python list-images.py ``` And the product image list by using the `--prod` flag: ```bash -$ python3 list-images.py --prod +$ python list-images.py --prod ``` diff --git a/scripts/common.py b/scripts/common.py index 40b8c777..f00377ff 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python # This script defines some common function that are used by manage-kogito-version.py and push-staging.py script diff --git a/scripts/list-images.py b/scripts/list-images.py index 7cfa9ede..e982d164 100644 --- a/scripts/list-images.py +++ b/scripts/list-images.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python # Script responsible to update the tests with # Should be run from root directory of the repository diff --git a/scripts/manage-kogito-version.py b/scripts/manage-kogito-version.py index ec344bc8..346215fa 100644 --- a/scripts/manage-kogito-version.py +++ b/scripts/manage-kogito-version.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python # This script will be responsible to help to manage kogito images and modules version, it will update all needed files # Example of usage: # # move the current version to the next one or rcX diff --git a/scripts/push-local-registry.sh b/scripts/push-local-registry.sh index 157eb11e..79d2ca89 100644 --- a/scripts/push-local-registry.sh +++ b/scripts/push-local-registry.sh @@ -26,5 +26,5 @@ while read image; do ${BUILD_ENGINE} tag quay.io/kiegroup/${image}:${version} ${registry}/${namespace}/${image}:${version} echo "Deleting imagestream ${image} if exists `oc delete oc -n ${namespace} ${image}`" ${BUILD_ENGINE} push ${registry}/${namespace}/${image}:${version} -done <<<$(python3 scripts/list-images.py) +done <<<$(python scripts/list-images.py) diff --git a/scripts/push-staging.py b/scripts/push-staging.py index 9a95c925..5fde3593 100644 --- a/scripts/push-staging.py +++ b/scripts/push-staging.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python # This script will be responsible to fetch the latest rc tags from each image and define the next # rc tag to avoid images get overridden. # diff --git a/scripts/retrieve_version.py b/scripts/retrieve_version.py index 64e4274c..b1b1c7a9 100644 --- a/scripts/retrieve_version.py +++ b/scripts/retrieve_version.py @@ -1,7 +1,7 @@ -#!/usr/bin/python3 +#!/usr/bin/env python #Script responsible to update the tests with #Should be run from root directory of the repository -#Sample usage: python3 scripts/retrieve_version.py +#Sample usage: python scripts/retrieve_version.py import sys sys.dont_write_bytecode = True diff --git a/scripts/update-repository.py b/scripts/update-repository.py index 0c632d1b..9103b468 100644 --- a/scripts/update-repository.py +++ b/scripts/update-repository.py @@ -1,7 +1,7 @@ -#!/usr/bin/python3 +#!/usr/bin/env python #Script responsible to update the tests with #Should be run from root directory of the repository -#Sample usage: python3 scripts/update-tests.py +#Sample usage: python scripts/update-tests.py import sys sys.dont_write_bytecode = True diff --git a/tests/test-apps/clone-repo.sh b/tests/test-apps/clone-repo.sh index f39a94bf..360d1d9f 100755 --- a/tests/test-apps/clone-repo.sh +++ b/tests/test-apps/clone-repo.sh @@ -8,7 +8,7 @@ if [ -n "${IMAGE_NAME}" ]; then if [[ ${IMAGE_NAME} =~ rhpam|logic* ]]; then prod="--prod" fi - if python3 ../../scripts/list-images.py ${prod} -is ${IMAGE_NAME}; then + if python ../../scripts/list-images.py ${prod} -is ${IMAGE_NAME}; then echo "Target image is supporting services, skipping examples build" exit 0 fi --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
