This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-29980 in repository https://gitbox.apache.org/repos/asf/hbase.git
commit aff01f2feaa2c63bef7b4e754af56eb69d877646 Author: Duo Zhang <[email protected]> AuthorDate: Mon Mar 9 10:15:57 2026 +0800 HBASE-29980 Polish jenkinsfiles after splitting integration test out Change to use pollSCM instead of cron in integration test. Remove hadoop 2 cache and hadoop 3 cache stages in nightly build as they are not used any more. --- dev-support/Jenkinsfile | 161 ++++++--------------- .../integration-test/integration-test.Jenkinsfile | 2 +- 2 files changed, 46 insertions(+), 117 deletions(-) diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 023d72cc79a..299370fca18 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -77,127 +77,56 @@ pipeline { } } stage ('thirdparty installs') { - parallel { - stage ('yetus install') { - steps { - // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :( - dir('downloads-yetus') { - // can't just do a simple echo or the directory won't be created. :( - sh '''#!/usr/bin/env bash - echo "Make sure we have a directory for downloading dependencies: $(pwd)" -''' - } - sh '''#!/usr/bin/env bash - set -e - echo "Ensure we have a copy of Apache Yetus." - if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then - YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}" - echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'" - if ! "${YETUS_DIR}/bin/test-patch" --version >/dev/null 2>&1 ; then - rm -rf "${YETUS_DIR}" - "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ - --working-dir "${WORKSPACE}/downloads-yetus" \ - --keys 'https://downloads.apache.org/yetus/KEYS' \ - --verify-tar-gz \ - "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \ - "yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz" - mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz - else - echo "Reusing cached install of Apache Yetus version ${YETUS_RELEASE}." - fi - else - YETUS_DIR="${WORKSPACE}/yetus-git" - rm -rf "${YETUS_DIR}" - echo "downloading from github" - curl -L --fail https://api.github.com/repos/apache/yetus/tarball/HEAD -o yetus.tar.gz - fi - if [ ! -d "${YETUS_DIR}" ]; then - echo "unpacking yetus into '${YETUS_DIR}'" - mkdir -p "${YETUS_DIR}" - gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1 - fi + stage ('yetus install') { + steps { + // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :( + dir('downloads-yetus') { + // can't just do a simple echo or the directory won't be created. :( + sh '''#!/usr/bin/env bash + echo "Make sure we have a directory for downloading dependencies: $(pwd)" ''' - // Set up the file we need at PERSONALITY_FILE location - dir ("tools") { - sh """#!/usr/bin/env bash - set -e - echo "Downloading Project personality from ${env.PROJECT_PERSONALITY}" - curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" - """ - } - stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*,tools/personality.sh" } - } - stage ('hadoop 2 cache') { - environment { - HADOOP2_VERSION="2.10.2" - } - steps { - // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :( - dir('downloads-hadoop-2') { - sh '''#!/usr/bin/env bash - echo "Make sure we have a directory for downloading dependencies: $(pwd)" -''' - } - sh '''#!/usr/bin/env bash + sh '''#!/usr/bin/env bash + set -e + echo "Ensure we have a copy of Apache Yetus." + if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then + YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}" + echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'" + if ! "${YETUS_DIR}/bin/test-patch" --version >/dev/null 2>&1 ; then + rm -rf "${YETUS_DIR}" + "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --working-dir "${WORKSPACE}/downloads-yetus" \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ + --verify-tar-gz \ + "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \ + "yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz" + mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz + else + echo "Reusing cached install of Apache Yetus version ${YETUS_RELEASE}." + fi + else + YETUS_DIR="${WORKSPACE}/yetus-git" + rm -rf "${YETUS_DIR}" + echo "downloading from github" + curl -L --fail https://api.github.com/repos/apache/yetus/tarball/HEAD -o yetus.tar.gz + fi + if [ ! -d "${YETUS_DIR}" ]; then + echo "unpacking yetus into '${YETUS_DIR}'" + mkdir -p "${YETUS_DIR}" + gunzip -c yetus.tar.gz | tar xpf - -C "${YETUS_DIR}" --strip-components 1 + fi + ''' + // Set up the file we need at PERSONALITY_FILE location + dir ("tools") { + sh """#!/usr/bin/env bash set -e - echo "Ensure we have a copy of Hadoop ${HADOOP2_VERSION}" - "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ - --working-dir "${WORKSPACE}/downloads-hadoop-2" \ - --keys 'https://downloads.apache.org/hadoop/common/KEYS' \ - --verify-tar-gz \ - "${WORKSPACE}/hadoop-${HADOOP2_VERSION}-bin.tar.gz" \ - "hadoop/common/hadoop-${HADOOP2_VERSION}/hadoop-${HADOOP2_VERSION}.tar.gz" - for stale in $(ls -1 "${WORKSPACE}"/hadoop-2*.tar.gz | grep -v ${HADOOP2_VERSION}); do - echo "Delete stale hadoop 2 cache ${stale}" - rm -rf $stale - done - ''' - stash name: 'hadoop-2', includes: "hadoop-${HADOOP2_VERSION}-bin.tar.gz" + echo "Downloading Project personality from ${env.PROJECT_PERSONALITY}" + curl -L -o personality.sh "${env.PROJECT_PERSONALITY}" + """ } + stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*,tools/personality.sh" } - stage ('hadoop 3 cache') { - steps { - script { - hadoop3_versions = env.HADOOP3_VERSIONS.split(","); - env.HADOOP3_VERSIONS_REGEX = "[" + hadoop3_versions.join("|") + "]"; - for (hadoop3_version in hadoop3_versions) { - env.HADOOP3_VERSION = hadoop3_version; - echo "env.HADOOP3_VERSION" + env.hadoop3_version; - stage ('Hadoop 3 cache inner stage') { - // directory must be unique for each parallel stage, because jenkins runs them in the same workspace :( - dir("downloads-hadoop-${HADOOP3_VERSION}") { - sh '''#!/usr/bin/env bash - echo "Make sure we have a directory for downloading dependencies: $(pwd)" -''' - } //dir - sh '''#!/usr/bin/env bash - set -e - echo "Ensure we have a copy of Hadoop ${HADOOP3_VERSION}" - "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ - --working-dir "${WORKSPACE}/downloads-hadoop-${HADOOP3_VERSION}" \ - --keys 'https://downloads.apache.org/hadoop/common/KEYS' \ - --verify-tar-gz \ - "${WORKSPACE}/hadoop-${HADOOP3_VERSION}-bin.tar.gz" \ - "hadoop/common/hadoop-${HADOOP3_VERSION}/hadoop-${HADOOP3_VERSION}.tar.gz" - for stale in $(ls -1 "${WORKSPACE}"/hadoop-3*.tar.gz | grep -v ${HADOOP3_VERSION}); do - echo "Delete stale hadoop 3 cache ${stale}" - rm -rf $stale - done - ''' - stash name: "hadoop-${HADOOP3_VERSION}", includes: "hadoop-${HADOOP3_VERSION}-bin.tar.gz" - script { - if (env.HADOOP3_VERSION == env.HADOOP3_DEFAULT_VERSION) { - // FIXME: we never unstash this, because we run the packaging tests with the version-specific stashes - stash(name: "hadoop-3", includes: "hadoop-${HADOOP3_VERSION}-bin.tar.gz") - } //if - } //script - } //stage ('Hadoop 3 cache inner stage') - } //for - } //script - } //steps - } //stage ('hadoop 3 cache') { - } //parallel + } } //stage ('thirdparty installs') stage ('init health results') { steps { diff --git a/dev-support/integration-test/integration-test.Jenkinsfile b/dev-support/integration-test/integration-test.Jenkinsfile index 5407ecda19a..d46f6d763b6 100644 --- a/dev-support/integration-test/integration-test.Jenkinsfile +++ b/dev-support/integration-test/integration-test.Jenkinsfile @@ -21,7 +21,7 @@ pipeline { } } triggers { - cron('@daily') + pollSCM('@daily') } options { buildDiscarder(logRotator(numToKeepStr: '20'))
