This is an automated email from the ASF dual-hosted git repository. zhangduo pushed a commit to branch HBASE-24899 in repository https://gitbox.apache.org/repos/asf/hbase-thirdparty.git
commit 35b67c60fad40b19435d417ed9b282496a585e3c Author: Duo Zhang <[email protected]> AuthorDate: Wed Aug 19 15:53:59 2020 +0800 HBASE-24899 [hbase-thirdparty] Add pre commit support for hbase-thirdparty --- dev-support/jenkins/Dockerfile | 36 ++++++ dev-support/jenkins/Jenkinsfile | 140 +++++++++++++++++++++ dev-support/jenkins/gather_machine_environment.sh | 58 +++++++++ .../jenkins/jenkins_precommit_github_yetus.sh | 124 ++++++++++++++++++ 4 files changed, 358 insertions(+) diff --git a/dev-support/jenkins/Dockerfile b/dev-support/jenkins/Dockerfile new file mode 100644 index 0000000..4351d66 --- /dev/null +++ b/dev-support/jenkins/Dockerfile @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for hbase-operator-tools pre-commit build. +# https://builds.apache.org/job/PreCommit-HBASE-OPERATOR-TOOLS-Build + +FROM maven:3.6-jdk-8 + +# hadolint ignore=DL3008 +RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ + git \ + rsync \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +CMD ["/bin/bash"] + +### +# Everything past this point is either not needed for testing or breaks Yetus. +# So tell Yetus not to read the rest of the file: +# YETUS CUT HERE +### diff --git a/dev-support/jenkins/Jenkinsfile b/dev-support/jenkins/Jenkinsfile new file mode 100644 index 0000000..92dc754 --- /dev/null +++ b/dev-support/jenkins/Jenkinsfile @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pipeline { + + agent { + label 'Hadoop' + } + + options { + // N.B. this is per-branch, which means per PR + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '15')) + timeout (time: 1, unit: 'HOURS') + timestamps() + skipDefaultCheckout() + } + + environment { + SRC_REL = 'src' + PATCH_REL = 'output' + YETUS_REL = 'yetus' + // Branch or tag name. Yetus release tags are 'rel/X.Y.Z' + YETUS_VERSION = 'rel/0.12.0' + DOCKERFILE_REL = "${SRC_REL}/dev-support/jenkins/Dockerfile" + YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins/jenkins_precommit_github_yetus.sh" + BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}" + SET_JAVA_HOME = '/usr/local/openjdk-8' + WORKDIR_REL = 'yetus-precommit-check' + WORKDIR = "${WORKSPACE}/${WORKDIR_REL}" + SOURCEDIR = "${WORKDIR}/${SRC_REL}" + PATCHDIR = "${WORKDIR}/${PATCH_REL}" + DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}" + YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}" + YETUSDIR = "${WORKDIR}/${YETUS_REL}" + PLUGINS = 'maven,asflicense,author,javac,mvnsite,github' + } + + parameters { + booleanParam(name: 'DEBUG', + defaultValue: false, + description: 'Print extra outputs for debugging the jenkins job and yetus') + } + + stages { + stage ('precommit checks') { + steps { + dir("${SOURCEDIR}") { + checkout scm + } + dir("${YETUSDIR}") { + checkout([ + $class : 'GitSCM', + branches : [[name: "${YETUS_VERSION}"]], + userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]] + ) + } + dir("${WORKDIR}") { + withCredentials([ + usernamePassword( + credentialsId: 'apache-hbase-at-github.com', + passwordVariable: 'GITHUB_PASSWORD', + usernameVariable: 'GITHUB_USER' + )]) { + sh label: 'test-patch', script: '''#!/bin/bash -e + printenv 2>&1 | sort + echo "[INFO] Launching Yetus via ${YETUS_DRIVER}" + "${YETUS_DRIVER}" + ''' + } + } + } + post { + always { + // Has to be relative to WORKSPACE. + archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit" + archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*" + publishHTML target: [ + allowMissing: true, + keepAll: true, + alwaysLinkToLastBuild: true, + // Has to be relative to WORKSPACE + reportDir: "${WORKDIR_REL}/${PATCH_REL}", + reportFiles: 'report.html', + reportName: 'PR General Check Report' + ] + } + // Jenkins pipeline jobs fill slaves on PRs without this :( + cleanup() { + script { + sh label: 'Cleanup workspace', script: '''#!/bin/bash -e + # See YETUS-764 + if [ -f "${PATCHDIR}/pidfile.txt" ]; then + echo "test-patch process appears to still be running: killing" + kill `cat "${PATCHDIR}/pidfile.txt"` || true + sleep 10 + fi + if [ -f "${PATCHDIR}/cidfile.txt" ]; then + echo "test-patch container appears to still be running: killing" + docker kill `cat "${PATCHDIR}/cidfile.txt"` || true + fi + # See HADOOP-13951 + chmod -R u+rxw "${WORKSPACE}" + ''' + dir ("${WORKDIR}") { + deleteDir() + } + } + } + } + } + } + + post { + // Jenkins pipeline jobs fill slaves on PRs without this :( + cleanup() { + script { + sh label: 'Cleanup workspace', script: '''#!/bin/bash -e + # See HADOOP-13951 + chmod -R u+rxw "${WORKSPACE}" + ''' + deleteDir() + } + } + } +} diff --git a/dev-support/jenkins/gather_machine_environment.sh b/dev-support/jenkins/gather_machine_environment.sh new file mode 100755 index 0000000..5edebe9 --- /dev/null +++ b/dev-support/jenkins/gather_machine_environment.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +function usage { + echo "Usage: ${0} /path/for/output/dir" + echo "" + echo " Gather info about a build machine that test harnesses should poll before running." + echo " presumes you'll then archive the passed output dir." + + exit 1 +} + +if [ "$#" -lt 1 ]; then + usage +fi + + +declare output=$1 + +if [ ! -d "${output}" ] || [ ! -w "${output}" ]; then + echo "Specified output directory must exist and be writable." >&2 + exit 1 +fi + +echo "getting machine specs, find in ${BUILD_URL}/artifact/${output}/" +echo "JAVA_HOME: ${JAVA_HOME}" >"${output}/java_home" 2>&1 || true +ls -l "${JAVA_HOME}" >"${output}/java_home_ls" 2>&1 || true +echo "MAVEN_HOME: ${MAVEN_HOME}" >"${output}/mvn_home" 2>&1 || true +mvn --offline --version >"${output}/mvn_version" 2>&1 || true +cat /proc/cpuinfo >"${output}/cpuinfo" 2>&1 || true +cat /proc/meminfo >"${output}/meminfo" 2>&1 || true +cat /proc/diskstats >"${output}/diskstats" 2>&1 || true +cat /sys/block/sda/stat >"${output}/sys-block-sda-stat" 2>&1 || true +df -h >"${output}/df-h" 2>&1 || true +ps -Aww >"${output}/ps-Aww" 2>&1 || true +ifconfig -a >"${output}/ifconfig-a" 2>&1 || true +lsblk -ta >"${output}/lsblk-ta" 2>&1 || true +lsblk -fa >"${output}/lsblk-fa" 2>&1 || true +ulimit -a >"${output}/ulimit-a" 2>&1 || true +uptime >"${output}/uptime" 2>&1 || true +hostname -a >"${output}/hostname-a" 2>&1 || true +© 2020 GitHub, Inc. diff --git a/dev-support/jenkins/jenkins_precommit_github_yetus.sh b/dev-support/jenkins/jenkins_precommit_github_yetus.sh new file mode 100644 index 0000000..e9c3a99 --- /dev/null +++ b/dev-support/jenkins/jenkins_precommit_github_yetus.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +# place ourselves in the directory containing the hbase and yetus checkouts +cd "$(dirname "$0")/../.." +echo "executing from $(pwd)" + +if [[ "true" = "${DEBUG}" ]]; then + set -x + printenv 2>&1 | sort +fi + +declare -i missing_env=0 +declare -a required_envs=( + # these ENV variables define the required API with Jenkinsfile_GitHub + "ARCHIVE_PATTERN_LIST" + "BUILD_URL_ARTIFACTS" + "DOCKERFILE" + "GITHUB_PASSWORD" + "GITHUB_USER" + "PATCHDIR" + "PLUGINS" + "SET_JAVA_HOME" + "SOURCEDIR" + "YETUSDIR" +) +# Validate params +for required_env in "${required_envs[@]}"; do + if [ -z "${!required_env}" ]; then + echo "[ERROR] Required environment variable '${required_env}' is not set." + missing_env=${missing_env}+1 + fi +done + +if [ ${missing_env} -gt 0 ]; then + echo "[ERROR] Please set the required environment variables before invoking. If this error is " \ + "on Jenkins, then please file a JIRA about the error." + exit 1 +fi + +# TODO (HBASE-23900): cannot assume test-patch runs directly from sources +TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh" + +# this must be clean for every run +rm -rf "${PATCHDIR}" +mkdir -p "${PATCHDIR}" + +# Gather machine information +mkdir "${PATCHDIR}/machine" +"${SOURCEDIR}/dev-support/jenkins/gather_machine_environment.sh" "${PATCHDIR}/machine" + +# If CHANGE_URL is set (e.g., Github Branch Source plugin), process it. +# Otherwise exit, because we don't want HBase to do a +# full build. We wouldn't normally do this check for smaller +# projects. :) +if [[ -z "${CHANGE_URL}" ]]; then + echo "Full build skipped" > "${PATCHDIR}/report.html" + exit 0 +fi +# enable debug output for yetus +if [[ "true" = "${DEBUG}" ]]; then + YETUS_ARGS+=("--debug") +fi +# If we're doing docker, make sure we don't accidentally pollute the image with a host java path +if [ -n "${JAVA_HOME}" ]; then + unset JAVA_HOME +fi +YETUS_ARGS+=("--patch-dir=${PATCHDIR}") +# where the source is located +YETUS_ARGS+=("--basedir=${SOURCEDIR}") +# lots of different output formats +YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt") +YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt") +YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html") +# enable writing back to Github +YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}") +YETUS_ARGS+=("--github-user=${GITHUB_USER}") +# auto-kill any surefire stragglers during unit test runs +YETUS_ARGS+=("--reapermode=kill") +# set relatively high limits for ASF machines +# changing these to higher values may cause problems +# with other jobs on systemd-enabled machines +YETUS_ARGS+=("--dockermemlimit=20g") +# rsync these files back into the archive dir +YETUS_ARGS+=("--archive-list=${ARCHIVE_PATTERN_LIST}") +# URL for user-side presentation in reports and such to our artifacts +YETUS_ARGS+=("--build-url-artifacts=${BUILD_URL_ARTIFACTS}") +# plugins to enable +YETUS_ARGS+=("--plugins=${PLUGINS}") +# run in docker mode and specifically point to our +# Dockerfile since we don't want to use the auto-pulled version. +YETUS_ARGS+=("--docker") +YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") +YETUS_ARGS+=("--mvn-custom-repos") +YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}") +# effectively treat dev-support as a custom maven module +YETUS_ARGS+=("--skip-dirs=dev-support") +# help keep the ASF boxes clean +YETUS_ARGS+=("--sentinel") +# use emoji vote so it is easier to find the broken line +YETUS_ARGS+=("--github-use-emoji-vote") +YETUS_ARGS+=("--github-repo=apache/hbase-thirdparty") + +echo "Launching yetus with command line:" +echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}" + +/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
