This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/master by this push:
new b6b5d42fe TEZ-4718: Modernize Jenkins and Yetus integration to ensure
full build and UT run on every PR (#498) (Raghav Aggarwal reviewed by Laszlo
Bodor)
b6b5d42fe is described below
commit b6b5d42fe3251dec42db31bc3bb0a09dc8cfd85e
Author: Raghav Aggarwal <[email protected]>
AuthorDate: Wed May 20 11:52:15 2026 +0530
TEZ-4718: Modernize Jenkins and Yetus integration to ensure full build and
UT run on every PR (#498) (Raghav Aggarwal reviewed by Laszlo Bodor)
---
Jenkinsfile | 272 ++++++-------
build-tools/smart-apply-patch.sh | 145 -------
build-tools/test-patch.sh | 821 ---------------------------------------
dev-support/tez-personality.sh | 118 ++++++
4 files changed, 241 insertions(+), 1115 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index b88da9bfc..bb507d4cf 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -23,188 +23,162 @@ pipeline {
options {
buildDiscarder(logRotator(numToKeepStr: '5'))
- timeout (time: 20, unit: 'HOURS')
+ timeout(time: 5, unit: 'HOURS')
timestamps()
checkoutToSubdirectory('src')
+ // Ensure only one build runs per PR at a time
+ disableConcurrentBuilds(abortPrevious: true)
+ }
+
+ parameters {
+ string(name: 'JIRA_ISSUE_KEY',
+ defaultValue: '',
+ description: 'The JIRA issue to comment on. Example: TEZ-1234')
}
environment {
SOURCEDIR = 'src'
- // will also need to change notification section below
+ YETUSDIR = 'yetus'
PATCHDIR = 'out'
DOCKERFILE = "${SOURCEDIR}/build-tools/docker/Dockerfile"
- YETUS='yetus'
- // Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
- YETUS_VERSION='rel/0.15.1'
- }
+ // Credentials IDs
+ GITHUB_CRED_ID = 'apache-tez-at-github.com'
+ JIRA_CRED_ID = 'tez-ci'
- parameters {
- string(name: 'JIRA_ISSUE_KEY',
- defaultValue: '',
- description: 'The JIRA issue that has a patch needing
pre-commit testing. Example: HADOOP-1234')
+ // Yetus version
+ YETUS_VERSION = 'rel/0.15.1'
}
stages {
- stage ('install yetus') {
+ stage('Install Yetus') {
steps {
- dir("${WORKSPACE}/${YETUS}") {
+ dir("${WORKSPACE}/${env.YETUSDIR}") {
checkout([
$class: 'GitSCM',
branches: [[name: "${env.YETUS_VERSION}"]],
- userRemoteConfigs: [[ url:
'https://github.com/apache/yetus']]]
+ userRemoteConfigs: [[url:
'https://github.com/apache/yetus']]]
)
}
}
}
- stage ('precommit-run') {
+ stage('Yetus Analysis') {
steps {
- withCredentials(
- [usernamePassword(credentialsId:
'apache-tez-at-github.com',
- passwordVariable: 'GITHUB_TOKEN',
- usernameVariable: 'GITHUB_USER'),
- usernamePassword(credentialsId:
'tez-ci',
- passwordVariable: 'JIRA_PASSWORD',
- usernameVariable: 'JIRA_USER')]) {
- sh '''#!/usr/bin/env bash
-
- set -e
-
-
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
-
- # this must be clean for every run
- if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
- rm -rf "${WORKSPACE}/${PATCHDIR}"
- fi
- mkdir -p "${WORKSPACE}/${PATCHDIR}"
-
- # if given a JIRA issue, process it. If CHANGE_URL is
set
- # (e.g., Github Branch Source plugin), process it.
- # otherwise exit, because we don't want Hadoop to do a
- # full build. We wouldn't normally do this check for
smaller
- # projects. :)
- if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
- YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
- elif [[ -z "${CHANGE_URL}" ]]; then
- echo "Full build skipped" >
"${WORKSPACE}/${PATCHDIR}/report.html"
- exit 0
- fi
-
- YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
-
- # where the source is located
- YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
-
- # our project defaults come from a personality file
- YETUS_ARGS+=("--project=tez")
-
- # lots of different output formats
-
YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
-
YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
-
YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
-
- # enable writing back to Github
- YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
-
- # auto-kill any surefire stragglers during unit test
runs
- YETUS_ARGS+=("--reapermode=kill")
-
- # set relatively high limits for ASF machines
- # changing these to higher values may cause problems
- # with other jobs on systemd-enabled machines
- YETUS_ARGS+=("--proclimit=5500")
- YETUS_ARGS+=("--dockermemlimit=20g")
-
- # -1 spotbugs issues that show up prior to the patch
being applied
- # YETUS_ARGS+=("--spotbugs-strict-precheck")
- # rsync these files back into the archive dir
-
YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")
-
- # URL for user-side presentation in reports and such
to our artifacts
- # (needs to match the archive bits below)
- YETUS_ARGS+=("--build-url-artifacts=artifact/out")
-
- # plugins to enable
- YETUS_ARGS+=("--plugins=all")
-
- # use Hadoop's bundled shelldocs
-
YETUS_ARGS+=("--shelldocs=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/shelldocs")
-
- # don't let these tests cause -1s because we aren't
really paying that
- # much attention to them
- YETUS_ARGS+=("--tests-filter=checkstyle")
-
- # run in docker mode and specifically point to our
- # Dockerfile since we don't want to use the
auto-pulled version.
- YETUS_ARGS+=("--docker")
- YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
- YETUS_ARGS+=("--mvn-custom-repos")
-
- # effectively treat dev-suport as a custom maven module
- YETUS_ARGS+=("--skip-dirs=dev-support")
-
- # help keep the ASF boxes clean
- YETUS_ARGS+=("--sentinel")
-
- # test with Java 21 eclipse-temurin docker image path
- YETUS_ARGS+=("--java-home=/opt/java/openjdk")
- YETUS_ARGS+=("--debug")
-
- # write Yetus report as GitHub comment (YETUS-1102)
- YETUS_ARGS+=("--github-write-comment")
- YETUS_ARGS+=("--github-use-emoji-vote")
-
- "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
- '''
- }
+ withCredentials([
+ usernamePassword(credentialsId: env.GITHUB_CRED_ID,
passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER'),
+ usernamePassword(credentialsId: env.JIRA_CRED_ID,
passwordVariable: 'JIRA_PASSWORD', usernameVariable: 'JIRA_USER')
+ ]) {
+ sh '''#!/usr/bin/env bash
+
+
TESTPATCHBIN="${WORKSPACE}/${YETUSDIR}/precommit/src/main/shell/test-patch.sh"
+
+ rm -rf "${WORKSPACE}/${PATCHDIR}"
+ mkdir -p "${WORKSPACE}/${PATCHDIR}"
+
+ cd "${WORKSPACE}/${SOURCEDIR}"
+ # Ensure origin/master is available for diffing
+ git fetch origin master || true
+ git diff origin/master...HEAD >
"${WORKSPACE}/${PATCHDIR}/local-pr.patch"
+ cd "${WORKSPACE}"
+
+ YETUS_ARGS=()
+
+ # Uncomment the following line to enable Yetus debugging
+ # YETUS_ARGS+=("--debug")
+
+ # 1. Target Issue/PR
+ if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
+ YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
+ elif [[ -n "${CHANGE_ID}" ]]; then
+ echo "Processing PR: ${CHANGE_ID} using local patch"
+ else
+ echo "No PR or JIRA key provided. Skipping Yetus."
+ exit 0
+ fi
+
+ # 2. Core Paths & Project
+ YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
+ YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
+ YETUS_ARGS+=("--project=tez")
+
+ # 3. Docker & Environment
+ YETUS_ARGS+=("--docker")
+ YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
+ YETUS_ARGS+=("--java-home=/opt/java/openjdk")
+ YETUS_ARGS+=("--build-url-artifacts=artifact/out")
+
+ # 4. Plugins & Personality
+ # Copy personality to PATCHDIR to avoid deletion by Yetus'
robot mode
+ cp
"${WORKSPACE}/${SOURCEDIR}/dev-support/tez-personality.sh"
"${WORKSPACE}/${PATCHDIR}/tez-personality.sh"
+
+ YETUS_ARGS+=("--plugins=all")
+
+
YETUS_ARGS+=("--personality=${WORKSPACE}/${PATCHDIR}/tez-personality.sh")
+ YETUS_ARGS+=("--mvn-custom-repos")
+ YETUS_ARGS+=("--tests-filter=checkstyle")
+
YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")
+ YETUS_ARGS+=("--reapermode=kill")
+ YETUS_ARGS+=("--sentinel")
+
+ # 5. Reporting & GitHub Integration
+ YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
+ YETUS_ARGS+=("--github-write-comment")
+ YETUS_ARGS+=("--github-use-emoji-vote")
+
YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
+
YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
+
YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
+
+ echo "Running Yetus with local patch overriding GitHub..."
+
+ "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
"${WORKSPACE}/${PATCHDIR}/local-pr.patch"
+ '''
+ }
}
}
-
}
post {
always {
- script {
- // Yetus output
- archiveArtifacts "${env.PATCHDIR}/**"
- // Publish the HTML report so that it can be looked at
- // Has to be relative to WORKSPACE.
- publishHTML (target: [
- allowMissing: true,
- keepAll: true,
- alwaysLinkToLastBuild: true,
- // Has to be relative to WORKSPACE
- reportDir: "${env.PATCHDIR}",
- reportFiles: 'report.html',
- reportName: 'Yetus Report'
- ])
- // Publish JUnit results
- try {
- junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
- } catch(e) {
- echo 'junit processing: ' + e.toString()
+ script {
+ sh '''
+ # Fix permissions before attempting to read/zip
directories created by root Docker processes
+ echo "Cleaning up workspace permissions to prevent
HADOOP-13951 and unzipping errors..."
+ chmod -R u+rxw "${WORKSPACE}" || true
+
+ # Zip test logs for easier debugging and storage efficiency
+ cd "${SOURCEDIR}" || exit 0
+ find . -type d -name "surefire-reports" | while read -r
dir; do
+ rel_path=$(echo "${dir#./}" | tr '/' '_')
+ zip -r -q "../${PATCHDIR}/test-logs-${rel_path}.zip"
"${dir}" || true
+ done
+ '''
+
+ dir(env.SOURCEDIR) {
+ // Safely publish junit results
+ junit testResults: "**/target/surefire-reports/*.xml",
allowEmptyResults: true
+ }
+
+ // Archive Yetus output, reports, and zipped test logs
+ archiveArtifacts artifacts: "${env.PATCHDIR}/**",
allowEmptyArchive: true
+
+ // Publish Yetus HTML report to Jenkins UI sidebar only if it
exists
+ if (fileExists("${env.PATCHDIR}/report.html")) {
+ publishHTML(target: [
+ allowMissing: true,
+ alwaysLinkToLastBuild: true,
+ keepAll: true,
+ reportDir: "${env.PATCHDIR}",
+ reportFiles: 'report.html',
+ reportName: 'Yetus Report'
+ ])
+ }
}
- }
}
- // Jenkins pipeline jobs fill slaves on PRs without this :(
- cleanup() {
+ cleanup {
script {
- sh '''
- # See YETUS-764
- if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
- echo "test-patch process appears to still be running:
killing"
- kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
- sleep 10
- fi
- if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
- echo "test-patch container appears to still be running:
killing"
- docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"`
|| true
- fi
- # See HADOOP-13951
- chmod -R u+rxw "${WORKSPACE}"
- '''
+ // Wipe the workspace to prevent Jenkins agent disk exhaustion
deleteDir()
}
}
diff --git a/build-tools/smart-apply-patch.sh b/build-tools/smart-apply-patch.sh
deleted file mode 100755
index 2a9dbd24d..000000000
--- a/build-tools/smart-apply-patch.sh
+++ /dev/null
@@ -1,145 +0,0 @@
-#!/usr/bin/env bash
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-#
-# Determine if the patch file is a git diff file with prefixes.
-# These files are generated via "git diff" *without* the --no-prefix option.
-#
-# We can apply these patches more easily because we know that the a/ and b/
-# prefixes in the "diff" lines stands for the project root directory.
-# So we don't have to hunt for the project root.
-# And of course, we know that the patch file was generated using git, so we
-# know git apply can handle it properly.
-#
-# Arguments: file name.
-# Return: 0 if it is a git diff; 1 otherwise.
-#
-is_git_diff_with_prefix() {
- DIFF_TYPE="unknown"
- while read -r line; do
- if [[ "$line" =~ ^diff\ ]]; then
- if [[ "$line" =~ ^diff\ \-\-git ]]; then
- DIFF_TYPE="git"
- else
- return 1 # All diff lines must be diff --git lines.
- fi
- fi
- if [[ "$line" =~ ^\+\+\+\ ]] ||
- [[ "$line" =~ ^\-\-\-\ ]]; then
- if ! [[ "$line" =~ ^....[ab]/ ]]; then
- return 1 # All +++ and --- lines must start with a/ or b/.
- fi
- fi
- done < $1
- [ x$DIFF_TYPE == x"git" ] || return 1
- return 0 # return true (= 0 in bash)
-}
-
-PATCH_FILE=$1
-DRY_RUN=$2
-if [ -z "$PATCH_FILE" ]; then
- echo usage: $0 patch-file
- exit 1
-fi
-
-PATCH=${PATCH:-patch} # allow overriding patch binary
-
-# Cleanup handler for temporary files
-TOCLEAN=""
-cleanup() {
- rm $TOCLEAN
- exit $1
-}
-trap "cleanup 1" HUP INT QUIT TERM
-
-# Allow passing "-" for stdin patches
-if [ "$PATCH_FILE" == "-" ]; then
- PATCH_FILE=/tmp/tmp.in.$$
- cat /dev/fd/0 > $PATCH_FILE
- TOCLEAN="$TOCLEAN $PATCH_FILE"
-fi
-
-# Special case for git-diff patches without --no-prefix
-if is_git_diff_with_prefix "$PATCH_FILE"; then
- GIT_FLAGS="--binary -p1 -v"
- if [[ -z $DRY_RUN ]]; then
- GIT_FLAGS="$GIT_FLAGS --stat --apply "
- echo Going to apply git patch with: git apply "${GIT_FLAGS}"
- else
- GIT_FLAGS="$GIT_FLAGS --check "
- fi
- git apply ${GIT_FLAGS} "${PATCH_FILE}"
- exit $?
-fi
-
-# Come up with a list of changed files into $TMP
-TMP=/tmp/tmp.paths.$$
-TOCLEAN="$TOCLEAN $TMP"
-
-if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then
- PLEVEL=0
- #if the patch applied at P0 there is the possability that all we are doing
- # is adding new files and they would apply anywhere. So try to guess the
- # correct place to put those files.
-
- TMP2=/tmp/tmp.paths.2.$$
- TOCLEAN="$TOCLEAN $TMP2"
-
- egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v
/dev/null | sort | uniq > $TMP2
-
- if [ ! -s $TMP2 ]; then
- echo "Error: Patch dryrun couldn't detect changes the patch would make.
Exiting."
- cleanup 1
- fi
-
- #first off check that all of the files do not exist
- FOUND_ANY=0
- for CHECK_FILE in $(cat $TMP2)
- do
- if [[ -f $CHECK_FILE ]]; then
- FOUND_ANY=1
- fi
- done
-
- if [[ "$FOUND_ANY" = "0" ]]; then
- #all of the files are new files so we have to guess where the correct
place to put it is.
-
- # if all of the lines start with a/ or b/, then this is a git patch that
- # was generated without --no-prefix
- if ! grep -qv '^a/\|^b/' $TMP2 ; then
- echo Looks like this is a git patch. Stripping a/ and b/ prefixes
- echo and incrementing PLEVEL
- PLEVEL=$[$PLEVEL + 1]
- sed -i -e 's,^[ab]/,,' $TMP2
- fi
- fi
-elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
- PLEVEL=1
-elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
- PLEVEL=2
-else
- echo "The patch does not appear to apply with p0 to p2";
- cleanup 1;
-fi
-
-# If this is a dry run then exit instead of applying the patch
-if [[ -n $DRY_RUN ]]; then
- cleanup 0;
-fi
-
-echo Going to apply patch with: $PATCH -p$PLEVEL
-$PATCH -p$PLEVEL -E < $PATCH_FILE
-
-cleanup $?
diff --git a/build-tools/test-patch.sh b/build-tools/test-patch.sh
deleted file mode 100755
index feb964489..000000000
--- a/build-tools/test-patch.sh
+++ /dev/null
@@ -1,821 +0,0 @@
-#!/usr/bin/env bash
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-#set -x
-
-### Setup some variables.
-### BUILD_URL is set by Hudson if it is run by patch process
-### Read variables from properties file
-bindir=$(dirname $0)
-
-# Defaults
-if [ -z "$MAVEN_HOME" ]; then
- MVN=mvn
-else
- MVN=$MAVEN_HOME/bin/mvn
-fi
-
-PROJECT_NAME=Tez
-JENKINS=false
-PATCH_DIR=/tmp
-BASEDIR=$(pwd)
-PS=${PS:-ps}
-AWK=${AWK:-awk}
-WGET=${WGET:-wget}
-GIT=${GIT:-git}
-GREP=${GREP:-grep}
-PATCH=${PATCH:-patch}
-DIFF=${DIFF:-diff}
-JIRACLI=${JIRA:-jira}
-SED=${SED:-sed}
-CURL=${CURL:-curl}
-SPOTBUGS_HOME=${SPOTBUGS_HOME}
-
-###############################################################################
-printUsage() {
- echo "Usage: $0 [options] patch-file | defect-number"
- echo
- echo "Where:"
- echo " patch-file is a local patch file containing the changes to test"
- echo " defect-number is a JIRA defect number (e.g. 'TEZ-1234') to test
(Jenkins only)"
- echo
- echo "Options:"
- echo "--patch-dir=<dir> The directory for working and output files
(default '/tmp')"
- echo "--basedir=<dir> The directory to apply the patch to (default
current directory)"
- echo "--mvn-cmd=<cmd> The 'mvn' command to use (default
\$MAVEN_HOME/bin/mvn, or 'mvn')"
- echo "--ps-cmd=<cmd> The 'ps' command to use (default 'ps')"
- echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')"
- echo "--git-cmd=<cmd> The 'git' command to use (default 'git')"
- echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')"
- echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')"
- echo "--diff-cmd=<cmd> The 'diff' command to use (default 'diff')"
- echo "--spotbugs-home=<path> Spotbugs home directory (default SPOTBUGS_HOME
environment variable)"
- echo "--dirty-workspace Allow the local git workspace to have
uncommitted changes"
- echo "--run-tests Run all tests below the base directory"
- echo
- echo "Jenkins-only options:"
- echo "--jenkins Run by Jenkins (runs tests and posts results to
JIRA)"
- echo "--wget-cmd=<cmd> The 'wget' command to use (default 'wget')"
- echo "--jira-cmd=<cmd> The 'jira' command to use (default 'jira')"
- echo "--jira-password=<pw> The password for the 'jira' command"
-}
-
-###############################################################################
-parseArgs() {
- for i in $*
- do
- case $i in
- --jenkins)
- JENKINS=true
- ;;
- --patch-dir=*)
- PATCH_DIR=${i#*=}
- ;;
- --basedir=*)
- BASEDIR=${i#*=}
- ;;
- --mvn-cmd=*)
- MVN=${i#*=}
- ;;
- --ps-cmd=*)
- PS=${i#*=}
- ;;
- --awk-cmd=*)
- AWK=${i#*=}
- ;;
- --wget-cmd=*)
- WGET=${i#*=}
- ;;
- --git-cmd=*)
- GIT=${i#*=}
- ;;
- --grep-cmd=*)
- GREP=${i#*=}
- ;;
- --patch-cmd=*)
- PATCH=${i#*=}
- ;;
- --diff-cmd=*)
- DIFF=${i#*=}
- ;;
- --jira-cmd=*)
- JIRACLI=${i#*=}
- ;;
- --jira-password=*)
- JIRA_PASSWD=${i#*=}
- ;;
- --spotbugs-home=*)
- SPOTBUGS_HOME=${i#*=}
- ;;
- --dirty-workspace)
- DIRTY_WORKSPACE=true
- ;;
- --run-tests)
- RUN_TESTS=true
- ;;
- *)
- PATCH_OR_DEFECT=$i
- ;;
- esac
- done
- if [ -z "$PATCH_OR_DEFECT" ]; then
- printUsage
- exit 1
- fi
- if [[ $JENKINS == "true" ]] ; then
- echo "Running in Jenkins mode"
- defect=$PATCH_OR_DEFECT
- else
- echo "Running in developer mode"
- JENKINS=false
- ### PATCH_FILE contains the location of the patchfile
- PATCH_FILE=$PATCH_OR_DEFECT
- if [[ ! -e "$PATCH_FILE" ]] ; then
- echo "Unable to locate the patch file $PATCH_FILE"
- cleanupAndExit 0
- fi
- ### Check if $PATCH_DIR exists. If it does not exist, create a new
directory
- if [[ ! -e "$PATCH_DIR" ]] ; then
- mkdir "$PATCH_DIR"
- if [[ $? == 0 ]] ; then
- echo "$PATCH_DIR has been created"
- else
- echo "Unable to create $PATCH_DIR"
- cleanupAndExit 0
- fi
- fi
- ### Obtain the patch filename to append it to the version number
- defect=`basename $PATCH_FILE`
- fi
-}
-
-###############################################################################
-checkout () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Testing patch for ${defect}."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- ### When run by a developer, if the workspace contains modifications, do not
continue
- ### unless the --dirty-workspace option was set
- status=`$GIT status --porcelain`
- if [[ $JENKINS == "false" ]] ; then
- if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then
- echo "ERROR: can't run in a workspace that contains the following
modifications"
- echo "$status"
- cleanupAndExit 1
- fi
- echo
- else
- cd $BASEDIR
- $GIT reset --hard
- $GIT clean -xdf
- $GIT checkout master
- $GIT pull --rebase
- fi
- GIT_REVISION=`git rev-parse --verify --short HEAD`
- return $?
-}
-
-###############################################################################
-downloadPatch () {
- ### Download latest patch file (ignoring .htm and .html) when run from patch
process
- if [[ $JENKINS == "true" ]] ; then
- $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect
- if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then
- echo "$defect is not \"Patch Available\". Exiting."
- cleanupAndExit 0
- fi
- relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*'
$PATCH_DIR/jira | $GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o
'/jira/secure/attachment/[0-9]*/[^"]*'`
- patchURL="http://issues.apache.org${relativePatchURL}";
- patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'`
- echo "$defect patch is being downloaded at `date` from"
- echo "$patchURL"
- $WGET -q -O $PATCH_DIR/patch $patchURL
- VERSION=${GIT_REVISION}_${defect}_PATCH-${patchNum}
- JIRA_COMMENT="Here are the results of testing the latest attachment
- $patchURL
- against master revision ${GIT_REVISION}."
-
- ### Copy the patch file to $PATCH_DIR
- else
- VERSION=PATCH-${defect}
- cp $PATCH_FILE $PATCH_DIR/patch
- if [[ $? == 0 ]] ; then
- echo "Patch file $PATCH_FILE copied to $PATCH_DIR"
- else
- echo "Could not copy $PATCH_FILE to $PATCH_DIR"
- cleanupAndExit 0
- fi
- fi
-}
-
-###############################################################################
-verifyPatch () {
- # Before building, check to make sure that the patch is valid
- $bindir/smart-apply-patch.sh $PATCH_DIR/patch dryrun
- if [[ $? != 0 ]] ; then
- echo "PATCH APPLICATION FAILED"
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 patch{color}. The patch command could not apply the patch."
- return 1
- else
- return 0
- fi
-}
-
-###############################################################################
-prebuildWithoutPatch () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Pre-build master to verify master stability and javac warnings"
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- echo "Compiling $(pwd)"
- echo "$MVN clean test -DskipTests -Ptest-patch >
$PATCH_DIR/masterJavacWarnings.txt 2>&1"
- $MVN clean test -DskipTests -Ptest-patch >
$PATCH_DIR/masterJavacWarnings.txt 2>&1
- if [[ $? != 0 ]] ; then
- echo "master compilation is broken?"
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 patch{color}. master compilation may be broken."
- return 1
- fi
-
- echo "$MVN clean test javadoc:javadoc -DskipTests >
$PATCH_DIR/masterJavadocWarnings.txt 2>&1"
- $MVN clean test javadoc:javadoc -DskipTests >
$PATCH_DIR/masterJavadocWarnings.txt 2>&1
- if [[ $? != 0 ]] ; then
- echo "master javadoc compilation is broken?"
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 patch{color}. master compilation may be broken."
- return 1
- fi
-
- return 0
-}
-
-###############################################################################
-### Check for @author tags in the patch
-checkAuthor () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Checking there are no @author tags in the patch."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- authorTags=`$GREP -c -i '@author' $PATCH_DIR/patch`
- echo "There appear to be $authorTags @author tags in the patch."
- if [[ $authorTags != 0 ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 @author{color}. The patch appears to contain $authorTags
@author tags which the Tez community has agreed to not allow in code
contributions."
- return 1
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 @author{color}. The patch does not contain any @author
tags."
- return 0
-}
-
-###############################################################################
-### Check for tests in the patch
-checkTests () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Checking there are new or changed tests in the patch."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- testReferences=`$GREP -c -i -e '^+++.*/test' $PATCH_DIR/patch`
- echo "There appear to be $testReferences test files referenced in the patch."
- if [[ $testReferences == 0 ]] ; then
- if [[ $JENKINS == "true" ]] ; then
- patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira`
- if [[ $patchIsDoc != 0 ]] ; then
- echo "The patch appears to be a documentation patch that doesn't
require tests."
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+0 tests included{color}. The patch appears to be a
documentation patch that doesn't require tests."
- return 0
- fi
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 tests included{color}. The patch doesn't appear to include
any new or modified tests.
- Please justify why no new tests are needed for this
patch.
- Also please list what manual steps were performed to
verify this patch."
- return 1
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 tests included{color}. The patch appears to include
$testReferences new or modified test files."
- return 0
-}
-
-cleanUpXml () {
- cd $BASEDIR/conf
- for file in `ls *.xml.template`
- do
- rm -f `basename $file .template`
- done
- cd $BASEDIR
-}
-
-###############################################################################
-### Attempt to apply the patch
-applyPatch () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Applying patch."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- export PATCH
- $bindir/smart-apply-patch.sh $PATCH_DIR/patch
- if [[ $? != 0 ]] ; then
- echo "PATCH APPLICATION FAILED"
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 patch{color}. The patch command could not apply the patch."
- return 1
- fi
- return 0
-}
-
-###############################################################################
-calculateJavadocWarnings() {
- WARNING_FILE="$1"
- RET=$(egrep "^[0-9]+ warnings?$" "$WARNING_FILE" | awk '{sum+=$1} END
{print sum}')
-}
-
-### Check there are no javadoc warnings
-checkJavadocWarnings () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Determining number of patched javadoc warnings."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- echo "$MVN clean test javadoc:javadoc -DskipTests >
$PATCH_DIR/patchJavadocWarnings.txt 2>&1"
- $MVN clean test javadoc:javadoc -DskipTests >
$PATCH_DIR/patchJavadocWarnings.txt 2>&1
- calculateJavadocWarnings "$PATCH_DIR/masterJavadocWarnings.txt"
- nummasterJavadocWarnings=$RET
- calculateJavadocWarnings "$PATCH_DIR/patchJavadocWarnings.txt"
- numPatchJavadocWarnings=$RET
- grep -i warning "$PATCH_DIR/masterJavadocWarnings.txt" >
"$PATCH_DIR/masterJavadocWarningsFiltered.txt"
- grep -i warning "$PATCH_DIR/patchJavadocWarnings.txt" >
"$PATCH_DIR/patchJavadocWarningsFiltered.txt"
- diff -u "$PATCH_DIR/masterJavadocWarningsFiltered.txt" \
- "$PATCH_DIR/patchJavadocWarningsFiltered.txt" > \
- "$PATCH_DIR/diffJavadocWarnings.txt"
- rm -f "$PATCH_DIR/masterJavadocWarningsFiltered.txt"
"$PATCH_DIR/patchJavadocWarningsFiltered.txt"
- echo "There appear to be $nummasterJavadocWarnings javadoc warnings before
the patch and $numPatchJavadocWarnings javadoc warnings after applying the
patch."
- if [[ $nummasterJavadocWarnings != "" && $numPatchJavadocWarnings != "" ]] ;
then
- if [[ $numPatchJavadocWarnings -gt $nummasterJavadocWarnings ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 javadoc{color}. The javadoc tool appears to have generated
`expr $(($numPatchJavadocWarnings-$nummasterJavadocWarnings))` warning messages.
- See $BUILD_URL/artifact/patchprocess/diffJavadocWarnings.txt for
details."
- return 1
- fi
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 javadoc{color}. There were no new javadoc warning
messages."
- return 0
-}
-
-###############################################################################
-### Check there are no changes in the number of Javac warnings
-checkJavacWarnings () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Determining number of patched javac warnings."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- echo "$MVN clean test -DskipTests -Ptest-patch >
$PATCH_DIR/patchJavacWarnings.txt 2>&1"
- $MVN clean test -DskipTests -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt
2>&1
- if [[ $? != 0 ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 javac{color:red}. The patch appears to cause the build to
fail."
- return 2
- fi
- ### Compare master and patch javac warning numbers
- if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then
- $GREP '\[WARNING\]' $PATCH_DIR/masterJavacWarnings.txt >
$PATCH_DIR/filteredmasterJavacWarnings.txt
- $GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt >
$PATCH_DIR/filteredPatchJavacWarnings.txt
- masterJavacWarnings=`cat $PATCH_DIR/filteredmasterJavacWarnings.txt | $AWK
'BEGIN {total = 0} {total += 1} END {print total}'`
- patchJavacWarnings=`cat $PATCH_DIR/filteredPatchJavacWarnings.txt | $AWK
'BEGIN {total = 0} {total += 1} END {print total}'`
- echo "There appear to be $masterJavacWarnings javac compiler warnings
before the patch and $patchJavacWarnings javac compiler warnings after applying
the patch."
- if [[ $patchJavacWarnings != "" && $masterJavacWarnings != "" ]] ; then
- if [[ $patchJavacWarnings -gt $masterJavacWarnings ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 javac{color}. The applied patch generated
$patchJavacWarnings javac compiler warnings (more than the master's current
$masterJavacWarnings warnings)."
-
- $DIFF $PATCH_DIR/filteredmasterJavacWarnings.txt
$PATCH_DIR/filteredPatchJavacWarnings.txt > $PATCH_DIR/diffJavacWarnings.txt
- JIRA_COMMENT_FOOTER="Javac warnings:
$BUILD_URL/artifact/patchprocess/diffJavacWarnings.txt
-$JIRA_COMMENT_FOOTER"
-
- return 1
- fi
- fi
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 javac{color}. The applied patch does not increase the
total number of javac compiler warnings."
- return 0
-}
-
-###############################################################################
-### Check there are no changes in the number of release audit (RAT) warnings
-checkReleaseAuditWarnings () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Determining number of patched release audit warnings."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- echo "$MVN apache-rat:check > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1"
- $MVN apache-rat:check > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1
- find $BASEDIR -name rat.txt | xargs cat >
$PATCH_DIR/patchReleaseAuditWarnings.txt
-
- ### Compare master and patch release audit warning numbers
- if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then
- patchReleaseAuditWarnings=`$GREP -c '\!?????'
$PATCH_DIR/patchReleaseAuditWarnings.txt`
- echo ""
- echo ""
- echo "There appear to be $patchReleaseAuditWarnings release audit warnings
after applying the patch."
- if [[ $patchReleaseAuditWarnings != "" ]] ; then
- if [[ $patchReleaseAuditWarnings -gt 0 ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 release audit{color}. The applied patch generated
$patchReleaseAuditWarnings release audit warnings."
- $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt >
$PATCH_DIR/patchReleaseAuditProblems.txt
- echo "Lines that start with ????? in the release audit report indicate
files that do not have an Apache license header." >>
$PATCH_DIR/patchReleaseAuditProblems.txt
- JIRA_COMMENT_FOOTER="Release audit warnings:
$BUILD_URL/artifact/patchprocess/patchReleaseAuditProblems.txt
-$JIRA_COMMENT_FOOTER"
- return 1
- fi
- fi
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 release audit{color}. The applied patch does not increase
the total number of release audit warnings."
- return 0
-}
-
-
-###############################################################################
-### Install the new jars so tests and spotbugs can find all of the updated jars
-buildAndInstall () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Installing all of the jars"
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- echo "$MVN install -Dmaven.javadoc.skip=true -DskipTests
-D${PROJECT_NAME}PatchProcess"
- $MVN install -Dmaven.javadoc.skip=true -DskipTests
-D${PROJECT_NAME}PatchProcess
- return $?
-}
-
-
-###############################################################################
-### Check there are no changes in the number of Spotbugs warnings
-checkSpotbugsWarnings () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Determining number of patched Spotbugs warnings."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
-
- rc=0
- echo " Running spotbugs "
- echo "$MVN clean test spotbugs:spotbugs -DskipTests < /dev/null >
$PATCH_DIR/patchSpotBugsOutput.txt 2>&1"
- $MVN clean test spotbugs:spotbugs -DskipTests < /dev/null >
$PATCH_DIR/patchSpotBugsOutput.txt 2>&1
- rc=$?
- spotbugs_version=$(${AWK} 'match($0, /spotbugs-maven-plugin:[^:]*:spotbugs/)
{ print substr($0, RSTART + 22, RLENGTH - 31); exit }'
"${PATCH_DIR}/patchSpotBugsOutput.txt")
-
- if [ $rc != 0 ] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 spotbugs{color}. The patch appears to cause Spotbugs
(version ${spotbugs_version}) to fail."
- return 1
- fi
-
- spotbugsWarnings=0
- for file in $(find $BASEDIR -name spotbugsXml.xml)
- do
- relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix
- if [ ! $relative_file == "target/spotbugsXml.xml" ]; then
- module_suffix=${relative_file%/target/spotbugsXml.xml} # strip trailing
path
- module_suffix=`basename ${module_suffix}`
- fi
-
- cp $file $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml
- $SPOTBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \
- $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \
- $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml
- newSpotbugsWarnings=`$SPOTBUGS_HOME/bin/filterBugs -first "01/01/2000"
$PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \
- $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml | $AWK '{print
$1}'`
- echo "Found $newSpotbugsWarnings Spotbugs warnings ($file)"
- spotbugsWarnings=$((spotbugsWarnings+newSpotbugsWarnings))
- $SPOTBUGS_HOME/bin/convertXmlToText -html \
- $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml \
- $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.html
- if [[ $newSpotbugsWarnings > 0 ]] ; then
- JIRA_COMMENT_FOOTER="Spotbugs warnings:
$BUILD_URL/artifact/patchprocess/newPatchSpotbugsWarnings${module_suffix}.html
-$JIRA_COMMENT_FOOTER"
- fi
- done
-
- if [[ $spotbugsWarnings -gt 0 ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:red}-1 spotbugs{color}. The patch appears to introduce
$spotbugsWarnings new Spotbugs (version ${spotbugs_version}) warnings."
- return 1
- fi
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 spotbugs{color}. The patch does not introduce any new
Spotbugs (version ${spotbugs_version}) warnings."
- return 0
-}
-
-###############################################################################
-### Run the tests
-runTests () {
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Running tests."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
-
- failed_tests=""
- failed_test_builds=""
- test_timeouts=""
- test_logfile=$PATCH_DIR/testrun.txt
- echo " Running tests "
- echo " $MVN clean install -fn -D${PROJECT_NAME}PatchProcess"
- $MVN clean install -fn > $test_logfile 2>&1
- test_build_result=$?
- cat $test_logfile
- module_test_timeouts=`$AWK '/^Running / { if (last) { print last } last=$2 }
/^Tests run: / { last="" }' $test_logfile`
- if [[ -n "$module_test_timeouts" ]] ; then
- test_timeouts="$test_timeouts $module_test_timeouts"
- fi
- module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E
"<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-|
|g" | sed -e "s|\.xml||g"`
- if [[ -n "$module_failed_tests" ]] ; then
- failed_tests="${failed_tests} ${module_failed_tests}"
- fi
- if [[ $test_build_result != 0 && -z "$module_failed_tests" && -z
"$module_test_timeouts" ]] ; then
- failed_test_builds="$module $failed_test_builds"
- fi
- result=0
- comment_prefix=" {color:red}-1 core tests{color}."
- if [[ -n "$failed_tests" ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
-$comment_prefix The patch failed these unit tests in $modules:
-$failed_tests"
- comment_prefix=" "
- result=1
- fi
- if [[ -n "$test_timeouts" ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
-$comment_prefix The following test timeouts occurred in $modules:
-$test_timeouts"
- comment_prefix=" "
- result=1
- fi
- if [[ -n "$failed_test_builds" ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
-$comment_prefix The test build failed in $failed_test_builds"
- result=1
- fi
- if [[ $result == 0 ]] ; then
- JIRA_COMMENT="$JIRA_COMMENT
-
- {color:green}+1 core tests{color}. The patch passed unit tests in
$modules."
- fi
- return $result
-}
-
-###############################################################################
-# Find the maven module containing the given file.
-findModule (){
- dir=`dirname $1`
- while [ 1 ]
- do
- if [ -f "$dir/pom.xml" ]
- then
- echo $dir
- return
- else
- dir=`dirname $dir`
- fi
- done
-}
-
-###############################################################################
-### Submit a comment to the defect's Jira
-submitJiraComment () {
- local result=$1
- ### Do not output the value of JIRA_COMMENT_FOOTER when run by a developer
- if [[ $JENKINS == "false" ]] ; then
- JIRA_COMMENT_FOOTER=""
- fi
- if [[ $result == 0 ]] ; then
- comment="{color:green}+1 overall{color}. $JIRA_COMMENT
-
-$JIRA_COMMENT_FOOTER"
- else
- comment="{color:red}-1 overall{color}. $JIRA_COMMENT
-
-$JIRA_COMMENT_FOOTER"
- fi
- ### Output the test result to the console
- echo "
-
-
-
-$comment"
-
- if [[ $JENKINS == "true" ]] ; then
- echo ""
- echo ""
- echo
"======================================================================"
- echo
"======================================================================"
- echo " Adding comment to Jira."
- echo
"======================================================================"
- echo
"======================================================================"
- echo ""
- echo ""
-
- # RESTify the comment
- jsoncomment=$(echo "$comment" \
- | ${SED} -e 's,\\,\\\\,g' \
- -e 's,\",\\\",g' \
- -e 's,$,\\r\\n,g' \
- | tr -d '\n')
- jsoncomment='{"body":"'"$jsoncomment"'"}'
-
- ### Update Jira with a comment
- ${CURL} -X POST \
- -H "Accept: application/json" \
- -H "Content-Type: application/json" \
- -u "tezqa:${JIRA_PASSWD}" \
- -d "$jsoncomment" \
- --silent --location \
- "https://issues.apache.org/jira/rest/api/2/issue/${defect}/comment"; \
- >/dev/null
- fi
-}
-
-###############################################################################
-### Cleanup files
-cleanupAndExit () {
- local result=$1
- if [[ $JENKINS == "true" ]] ; then
- if [ -e "$PATCH_DIR" ] ; then
- mv $PATCH_DIR $BASEDIR
- fi
- fi
- echo ""
- echo ""
- echo "======================================================================"
- echo "======================================================================"
- echo " Finished build."
- echo "======================================================================"
- echo "======================================================================"
- echo ""
- echo ""
- exit $result
-}
-
-###############################################################################
-###############################################################################
-###############################################################################
-
-JIRA_COMMENT=""
-JIRA_COMMENT_FOOTER="Console output: $BUILD_URL/console
-
-This message is automatically generated."
-
-### Check if arguments to the script have been specified properly or not
-parseArgs $@
-cd $BASEDIR
-
-checkout
-RESULT=$?
-if [[ $JENKINS == "true" ]] ; then
- if [[ $RESULT != 0 ]] ; then
- exit 100
- fi
-fi
-downloadPatch
-verifyPatch
-(( RESULT = RESULT + $? ))
-if [[ $RESULT != 0 ]] ; then
- submitJiraComment 1
- cleanupAndExit 1
-fi
-prebuildWithoutPatch
-(( RESULT = RESULT + $? ))
-if [[ $RESULT != 0 ]] ; then
- submitJiraComment 1
- cleanupAndExit 1
-fi
-checkAuthor
-(( RESULT = RESULT + $? ))
-
-if [[ $JENKINS == "true" ]] ; then
- cleanUpXml
-fi
-checkTests
-(( RESULT = RESULT + $? ))
-applyPatch
-APPLY_PATCH_RET=$?
-(( RESULT = RESULT + $APPLY_PATCH_RET ))
-if [[ $APPLY_PATCH_RET != 0 ]] ; then
- submitJiraComment 1
- cleanupAndExit 1
-fi
-checkJavacWarnings
-JAVAC_RET=$?
-#2 is returned if the code could not compile
-if [[ $JAVAC_RET == 2 ]] ; then
- submitJiraComment 1
- cleanupAndExit 1
-fi
-(( RESULT = RESULT + $JAVAC_RET ))
-checkJavadocWarnings
-(( RESULT = RESULT + $? ))
-buildAndInstall
-checkSpotbugsWarnings
-(( RESULT = RESULT + $? ))
-checkReleaseAuditWarnings
-(( RESULT = RESULT + $? ))
-### Run tests for Jenkins or if explictly asked for by a developer
-if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then
- runTests
- (( RESULT = RESULT + $? ))
-fi
-JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/
-$JIRA_COMMENT_FOOTER"
-
-submitJiraComment $RESULT
-cleanupAndExit $RESULT
diff --git a/dev-support/tez-personality.sh b/dev-support/tez-personality.sh
new file mode 100755
index 000000000..7e204c9eb
--- /dev/null
+++ b/dev-support/tez-personality.sh
@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Apache Yetus personality for Apache Tez
+
+personality_plugins "all"
+
+## @description Globals for this personality
+## @audience private
+## @stability evolving
+## @replaceable no
+function personality_globals {
+ export PROJECT_NAME=tez
+ export BUILDTOOL=maven
+ export PATCH_BRANCH_DEFAULT=master
+ export JIRA_ISSUE_RE='^TEZ-[0-9]+$'
+ export GITHUB_REPO="apache/tez"
+
+ # Increase memory for maven
+ export MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx4g -XX:+UseG1GC"}"
+
+ # Default Yetus settings
+ export DOCKER_MEMORY="20g"
+ export PROC_LIMIT=5500
+
+ # Force-enable core verification plugins
+ add_test compile
+ add_test unit
+}
+
+## @description Module selection
+## @audience private
+## @stability evolving
+## @replaceable no
+function personality_modules {
+ local repostatus=$1
+ local testtype=$2
+ local extra=()
+ local MODULES=("${CHANGED_MODULES[@]}")
+
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ extra=("-Ptools")
+
+ # Apply strict linting profile only during the explicit compile phase.
+ # Yetus ignores warnings during mvninstall, and unit tests are too slow with
strict linting.
+ if [[ ${testtype} == "compile" ]]; then
+ extra+=("-Ptest-patch")
+ fi
+
+ # Execute core compilation, unit tests, and install globally (project root)
+ if [[ ${testtype} == "unit" || ${testtype} == "compile" || ${testtype} ==
"mvninstall" ]]; then
+ yetus_debug "Forcing root module for ${testtype}"
+ MODULES=(.)
+ fi
+
+ # Stream unit test logs live to the Jenkins console to debug infinite
deadlocks
+ if [[ ${testtype} == "unit" && ${repostatus} == "patch" ]]; then
+ yetus_debug "Starting background tail for patch unit tests"
+ touch "${PATCH_DIR}/patch-unit-root.txt"
+ tail -f "${PATCH_DIR}/patch-unit-root.txt" &
+ export TAIL_PID=$!
+ # Trap to ensure tail is killed when Yetus finishes
+ trap 'kill ${TAIL_PID} 2>/dev/null || true' EXIT
+ fi
+
+ if [[ ${testtype} == "spotbugs" ]]; then
+ extra+=("-Pspotbugs")
+ fi
+
+ for module in "${MODULES[@]}"; do
+ personality_enqueue_module "${module}" "${extra[@]}"
+ done
+}
+
+## @description File filter to determine which tests to run
+## @audience private
+## @stability evolving
+## @replaceable no
+function personality_file_filter {
+ local filename=$1
+
+ yetus_debug "Tez file filter: ${filename}"
+
+ if [[ ${filename} =~ \.java$ || ${filename} =~ \.proto$ || ${filename} =~
pom\.xml$ ]]; then
+ add_test javac
+ add_test spotbugs
+ add_test checkstyle
+ add_test javadoc
+ fi
+
+ if [[ ${filename} =~ findbugs-exclude\.xml$ ]]; then
+ add_test spotbugs
+ fi
+
+ if [[ ${filename} =~ \.sh$ ]] || [[ ${filename} =~ Jenkinsfile ]]; then
+ add_test shellcheck
+ fi
+
+ if [[ ${filename} =~ \.md$ ]] || [[ ${filename} =~ \.txt$ ]]; then
+ add_test codespell
+ fi
+}
