This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch 4.x
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x by this push:
new 79027f6 PHOENIX-5032 add Apache Yetus to Phoenix
79027f6 is described below
commit 79027f6f055dd7d12049297838a8434ebabe97a1
Author: Istvan Toth <st...@apache.org>
AuthorDate: Thu Sep 3 07:00:54 2020 +0200
PHOENIX-5032 add Apache Yetus to Phoenix
also replace findbugs support with spotbugs support in maven build
and get mvn site working, and add all reports to it
---
BUILDING.md | 21 +-
Jenkinsfile.yetus | 66 ++++++
dev/docker/Dockerfile.yetus | 16 ++
dev/jenkins_precommit_jira_yetus.sh | 177 ++++++++++++++++
dev/phoenix-personality.sh | 363 +++++++++++++++++++++++++++++++++
phoenix-core/pom.xml | 52 -----
phoenix-hbase-compat-1.3.0/pom.xml | 12 --
phoenix-hbase-compat-1.4.0/bin/pom.xml | 83 --------
phoenix-hbase-compat-1.4.0/pom.xml | 12 --
phoenix-hbase-compat-1.5.0/pom.xml | 12 --
pom.xml | 131 ++++++++----
11 files changed, 719 insertions(+), 226 deletions(-)
diff --git a/BUILDING.md b/BUILDING.md
index 57116bd..97ee769 100644
--- a/BUILDING.md
+++ b/BUILDING.md
@@ -79,7 +79,7 @@ Running the tests
All Unit tests
`$ mvn clean test`
-All Unit Tests and Integration tests
+All Unit Tests and Integration tests (takes a few hours)
`$ mvn clean verify`
The verify maven target will also run dependency:analyze-only, which checks if
the dependencies
@@ -89,19 +89,20 @@ generated at /target/site/jacoco/index.html
To skip code coverage analysis
`$ mvn verify -Dskip.code-coverage`
-Running OWASP Dependency-Check
-------------------------------
+Running project reports
+-----------------------
-To run OWASP Dependency-Check (https://owasp.org/www-project-dependency-check/)
-`$ mvn verify -DskipTests -Dowasp-check`
+Phoenix currently supports generating the standard set of Maven Project Info
Reports, as well as
+Spotbugs, Apache Creadur RAT, OWASP Dependency-Check, and Jacoco Code Coverage
reports.
-The report is generated in target/dependency-check-report.html
+To run all available reports (takes a few hours)
+`$ mvn clean verify site -Dspotbugs.site`
-Findbugs
---------
+To run OWASP, RAT and Spotbugs, but not Jacoco (takes ~10 minutes)
+`$ mvn clean compile test-compile site -Dspotbugs.site`
-Findbugs report is generated in /target/site
-`$ mvn site`
+The reports are accessible via `target/site/index.html`, under the main
project,
+as well as each of the subprojects. (not every project has all reports)
Generate Apache Web Site
------------------------
diff --git a/Jenkinsfile.yetus b/Jenkinsfile.yetus
new file mode 100644
index 0000000..6e77526
--- /dev/null
+++ b/Jenkinsfile.yetus
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+pipeline {
+ agent {
+ label 'Hadoop'
+ }
+
+ options {
+ buildDiscarder(logRotator(daysToKeepStr: '7'))
+ timestamps()
+ }
+
+ stages {
+
+ stage('Yetus') {
+
+ environment {
+ MAVEN_OPTS = '-Xmx3G'
+ DEBUG = 'true'
+ RUN_IN_DOCKER = 'true'
+ }
+
+ options {
+ timeout(time: 5, unit: 'HOURS')
+ }
+
+ steps {
+ sh "component/dev/jenkins_precommit_jira_yetus.sh"
+ }
+
+ post {
+ always {
+ junit testResults: "component/**/target/**/TEST-*.xml",
allowEmptyResults: true
+ // Has to be relative to WORKSPACE.
+ archiveArtifacts artifacts: "patchprocess/*", excludes:
"patchprocess/precommit"
+ archiveArtifacts artifacts: "patchprocess/**/*", excludes:
"patchprocess/precommit/**/*"
+ publishHTML target: [
+ allowMissing: true,
+ keepAll: true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE
+ reportDir: "/",
+ reportFiles: 'report.html',
+ reportName: 'Precommit Report'
+ ]
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/dev/docker/Dockerfile.yetus b/dev/docker/Dockerfile.yetus
new file mode 100644
index 0000000..4ac1bea
--- /dev/null
+++ b/dev/docker/Dockerfile.yetus
@@ -0,0 +1,16 @@
+FROM apache/yetus:0.12.0
+
+#####
+# Update SpotBugs
+#####
+RUN rm -rf /opt/spotbugs \
+ && git clone https://github.com/stoty/spotbugs.git \
+ && cd spotbugs \
+ && git checkout PHOENIX-1161-backport \
+ && ./gradlew clean build -x test \
+ && mkdir /opt/spotbugs \
+ && tar -C /opt/spotbugs --strip-components 1 -xpf
spotbugs/build/distributions/spotbugs-4.1.2.tgz \
+ && chmod a+rx /opt/spotbugs/bin/* \
+ && cd .. \
+ && rm -rf ./spotbugs
+ENV SPOTBUGS_HOME /opt/spotbugs
diff --git a/dev/jenkins_precommit_jira_yetus.sh
b/dev/jenkins_precommit_jira_yetus.sh
new file mode 100755
index 0000000..5a071e8
--- /dev/null
+++ b/dev/jenkins_precommit_jira_yetus.sh
@@ -0,0 +1,177 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#
+# Based on jenkins_precommit_jira_yetus.sh of the HBase project
+#
+
+if [[ "true" = "${DEBUG}" ]]; then
+ set -x
+ printenv
+fi
+
+##To set jenkins Environment Variables:
+export TOOLS_HOME=/home/jenkins/tools
+#export JAVA_HOME=${JAVA_HOME_HADOOP_MACHINES_HOME}
+export CLOVER_HOME=${TOOLS_HOME}/clover/latest
+#export MAVEN_HOME=${MAVEN_3_0_4_HOME}
+export MAVEN_HOME=/home/jenkins/tools/maven/apache-maven-3.0.5
+
+#export PATH=$PATH:${JAVA_HOME}/bin:${MAVEN_HOME}/bin:
+export PATH=$PATH:${MAVEN_HOME}/bin:
+
+YETUS_RELEASE=0.12.0
+COMPONENT=${WORKSPACE}/component
+TEST_FRAMEWORK=${WORKSPACE}/test_framework
+
+PATCHPROCESS=${WORKSPACE}/patchprocess
+if [[ -d ${PATCHPROCESS} ]]; then
+ echo "[WARN] patch process already existed '${PATCHPROCESS}'"
+ rm -rf "${PATCHPROCESS}"
+fi
+mkdir -p "${PATCHPROCESS}"
+
+
+## Checking on H* machine nonsense
+echo "JAVA_HOME: ${JAVA_HOME}"
+ls -l "${JAVA_HOME}" || true
+echo "MAVEN_HOME: ${MAVEN_HOME}"
+echo "maven version:"
+mvn --offline --version || true
+echo "getting machine specs, find in
${BUILD_URL}/artifact/patchprocess/machine/"
+mkdir "${PATCHPROCESS}/machine"
+cat /proc/cpuinfo >"${PATCHPROCESS}/machine/cpuinfo" 2>&1 || true
+cat /proc/meminfo >"${PATCHPROCESS}/machine/meminfo" 2>&1 || true
+cat /proc/diskstats >"${PATCHPROCESS}/machine/diskstats" 2>&1 || true
+cat /sys/block/sda/stat >"${PATCHPROCESS}/machine/sys-block-sda-stat" 2>&1 ||
true
+df -h >"${PATCHPROCESS}/machine/df-h" 2>&1 || true
+ps -Awwf >"${PATCHPROCESS}/machine/ps-Awwf" 2>&1 || true
+ifconfig -a >"${PATCHPROCESS}/machine/ifconfig-a" 2>&1 || true
+lsblk -ta >"${PATCHPROCESS}/machine/lsblk-ta" 2>&1 || true
+lsblk -fa >"${PATCHPROCESS}/machine/lsblk-fa" 2>&1 || true
+cat /proc/loadavg >"${PATCHPROCESS}/loadavg" 2>&1 || true
+ulimit -a >"${PATCHPROCESS}/machine/ulimit-a" 2>&1 || true
+
+## /H*
+
+### Download Yetus
+if [[ "true" != "${USE_YETUS_PRERELEASE}" ]]; then
+ if [ ! -d "${TEST_FRAMEWORK}/apache-yetus-${YETUS_RELEASE}" ]; then
+ mkdir -p "${TEST_FRAMEWORK}"
+ cd "${TEST_FRAMEWORK}" || exit 1
+ # clear out any cached 'use a prerelease' versions
+ rm -rf apache-yetus-*
+
+ mkdir -p "${TEST_FRAMEWORK}/.gpg"
+ chmod -R 700 "${TEST_FRAMEWORK}/.gpg"
+
+ curl -L --fail -o "${TEST_FRAMEWORK}/KEYS_YETUS"
https://dist.apache.org/repos/dist/release/yetus/KEYS
+ gpg --homedir "${TEST_FRAMEWORK}/.gpg" --import
"${TEST_FRAMEWORK}/KEYS_YETUS"
+
+ ## Release
+ curl -L --fail -O
"https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz";
+ curl -L --fail -O
"https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz.asc";
+ gpg --homedir "${TEST_FRAMEWORK}/.gpg" --verify
"apache-yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
+ tar xzpf "apache-yetus-${YETUS_RELEASE}-bin.tar.gz"
+ fi
+ TESTPATCHBIN=${TEST_FRAMEWORK}/apache-yetus-${YETUS_RELEASE}/bin/test-patch
+ TESTPATCHLIB=${TEST_FRAMEWORK}/apache-yetus-${YETUS_RELEASE}/lib/precommit
+else
+ prerelease_dirs=("${TEST_FRAMEWORK}/${YETUS_PRERELEASE_GITHUB/\//-}-*")
+ if [ ! -d "${prerelease_dirs[0]}" ]; then
+ mkdir -p "${TEST_FRAMEWORK}"
+ cd "${TEST_FRAMEWORK}" || exit
+ ## from github
+ curl -L --fail
"https://api.github.com/repos/${YETUS_PRERELEASE_GITHUB}/tarball/HEAD"; >
yetus.tar.gz
+ tar xvpf yetus.tar.gz
+ prerelease_dirs=("${TEST_FRAMEWORK}/${YETUS_PRERELEASE_GITHUB/\//-}-*")
+ fi
+ TESTPATCHBIN=${prerelease_dirs[0]}/precommit/test-patch.sh
+ TESTPATCHLIB=${prerelease_dirs[0]}/precommit
+fi
+
+if [[ "true" = "${DEBUG}" ]]; then
+ # DEBUG print the test framework
+ ls -l "${TESTPATCHBIN}"
+ ls -la "${TESTPATCHLIB}/test-patch.d/"
+ # DEBUG print the local customization
+ if [ -d "${COMPONENT}/dev/test-patch.d" ]; then
+ ls -la "${COMPONENT}/dev/test-patch.d/"
+ fi
+ YETUS_ARGS=(--debug "${YETUS_ARGS[@]}")
+fi
+
+if [ -r "${COMPONENT}/dev/phoenix-personality.sh" ] ; then
+ YETUS_ARGS=("--personality=${COMPONENT}/dev/phoenix-personality.sh"
"${YETUS_ARGS[@]}")
+fi
+
+if [ ! -x "${TESTPATCHBIN}" ] && [ -n "${TEST_FRAMEWORK}" ] && [ -d
"${TEST_FRAMEWORK}" ]; then
+ echo "Something is amiss with the test framework; removing it. please
re-run."
+ rm -rf "${TEST_FRAMEWORK}"
+ exit 1
+fi
+
+cd "${WORKSPACE}" || exit
+
+
+#
+# Yetus *always* builds with JAVA_HOME, so no need to list it.
+#
+# non-docker-mode JDK:
+
+# docker-mode: (openjdk 7 added for free)
+# --docker \
+# --multijdkdirs="/usr/lib/jvm/java-8-openjdk-amd64" \
+
+if [[ "true" = "${RUN_IN_DOCKER}" ]]; then
+ YETUS_ARGS=(
+ --docker \
+ --dockermemlimit=20g \
+ "--multijdkdirs=/usr/lib/jvm/java-8-openjdk-amd64" \
+ "${YETUS_ARGS[@]}" \
+ )
+ if [ -r "${COMPONENT}/dev/docker/Dockerfile.yetus" ]; then
+ YETUS_ARGS=("--dockerfile=${COMPONENT}/dev/docker/Dockerfile.yetus"
"${YETUS_ARGS[@]}")
+ fi
+fi
+
+if [ -d "${COMPONENT}/dev/test-patch.d" ]; then
+ YETUS_ARGS=("--user-plugins=${COMPONENT}/dev/test-patch.d"
"${YETUS_ARGS[@]}")
+fi
+
+#Removed some code not relevant to Phoenix
+
+YETUS_ARGS=("--skip-dirs=dev" "${YETUS_ARGS[@]}")
+
+
+/bin/bash "${TESTPATCHBIN}" \
+ "${YETUS_ARGS[@]}" \
+ --patch-dir="${PATCHPROCESS}" \
+ --basedir="${COMPONENT}" \
+ --mvn-custom-repos \
+ --mvn-custom-repos-dir="${WORKSPACE}/yetus-m2/" \
+ --whitespace-eol-ignore-list=".*/generated/.*" \
+ --whitespace-tabs-ignore-list=".*/generated/.*" \
+ --run-tests \
+ --plugins="all,-findbugs,-gitlab" \
+ --html-report-file=report.html \
+ --jira-user=hadoopqa \
+ --jira-password="${JIRA_PASSWORD}" \
+ "PHOENIX-${ISSUE_NUM}"
+
+find "${COMPONENT}" -name target -exec chmod -R u+w {} \;
diff --git a/dev/phoenix-personality.sh b/dev/phoenix-personality.sh
new file mode 100755
index 0000000..b3b7791
--- /dev/null
+++ b/dev/phoenix-personality.sh
@@ -0,0 +1,363 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Based on hbase-personality.sh of the HBase project
+#
+# You'll need a local installation of
+# [Apache Yetus' precommit
checker](http://yetus.apache.org/documentation/0.12.0/#yetus-precommit)
+# to use this personality.
+#
+# Download from: http://yetus.apache.org/downloads/ . You can either grab the
source artifact and
+# build from it, or use the convenience binaries provided on that download
page.
+#
+# To run against, e.g. PHOENIX-5032 you'd then do
+# ```bash
+# test-patch --personality=dev-support/hbase-personality.sh HBASE-15074
+# ```
+#
+# pass the `--sentinel` flag if you want to allow test-patch to destructively
alter local working
+# directory / branch in order to have things match what the issue patch
requests.
+
+personality_plugins "all"
+
+if ! declare -f "yetus_info" >/dev/null; then
+
+ function yetus_info
+ {
+ echo "[$(date) INFO]: $*" 1>&2
+ }
+
+fi
+
+# work around yetus overwriting JAVA_HOME from our docker image
+#function docker_do_env_adds
+#{
+# declare k
+# for k in "${DOCKER_EXTRAENVS[@]}"; do
+# if [[ "JAVA_HOME" == "${k}" ]]; then
+# if [ -n "${JAVA_HOME}" ]; then
+# DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
+# fi
+# else
+# DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
+# fi
+# done
+#}
+
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ BUILDTOOL=maven
+ #shellcheck disable=SC2034
+ PROJECT_NAME=phoenix
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=master
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^PHOENIX-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/phoenix"
+
+ # TODO use PATCH_BRANCH to select jdk versions to use.
+
+ # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
+ # Up it. See HBASE-19902 for how we arrived at this number.
+ #shellcheck disable=SC2034
+ PROC_LIMIT=12500
+
+ # Set docker container to run with 20g. Default is 4g in yetus.
+ # See HBASE-19902 for how we arrived at 20g.
+ # TODO Doesn't seem to have effect in Yetus 0.12, set in cli instead
+ #shellcheck disable=SC2034
+ DOCKERMEMLIMIT=20g
+}
+
+## @description Parse extra arguments required by personalities, if any.
+## @audience private
+## @stability evolving
+function personality_parse_args
+{
+ declare i
+
+ for i in "$@"; do
+ case ${i} in
+ --exclude-tests-url=*)
+ delete_parameter "${i}"
+ EXCLUDE_TESTS_URL=${i#*=}
+ ;;
+ --include-tests-url=*)
+ delete_parameter "${i}"
+ INCLUDE_TESTS_URL=${i#*=}
+ ;;
+ --hbase-profile=*)
+ delete_parameter "${i}"
+ HBASE_PROFILE=${i#*=}
+ ;;
+ --skip-errorprone)
+ delete_parameter "${i}"
+ SKIP_ERRORPRONE=true
+ ;;
+ esac
+ done
+}
+
+## @description Queue up modules for this personality
+## @audience private
+## @stability evolving
+## @param repostatus
+## @param testtype
+function personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+ local extra=""
+ local jdk8module=""
+ local MODULES=("${CHANGED_MODULES[@]}")
+
+ yetus_info "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ # Running with threads>1 seems to trigger some problem in the build, but
since we
+ # spend 80+% of the time in phoenix-core, it wouldn't help much anyway
+ extra="--threads=1 -DPhoenixPatchProcess"
+ if [[ "${PATCH_BRANCH}" = 4* ]]; then
+ extra="${extra} -Dhttps.protocols=TLSv1.2"
+ fi
+
+ # If we have HBASE_PROFILE specified pass along
+ # the hadoop.profile system property.
+ if [[ -n "${HBASE_PROFILE}" ]] ; then
+ extra="${extra} -Dhbase.profile=${HBASE_PROFILE}"
+ fi
+
+ # BUILDMODE value is 'full' when there is no patch to be tested, and we are
running checks on
+ # full source code instead. In this case, do full compiles, tests, etc
instead of per
+ # module.
+ # Used in nightly runs.
+ # If BUILDMODE is 'patch', for unit and compile testtypes, there is no need
to run individual
+ # modules if root is included. HBASE-18505
+ if [[ "${BUILDMODE}" == "full" ]] || \
+ { { [[ "${testtype}" == unit ]] || [[ "${testtype}" == compile ]] || [[
"${testtype}" == checkstyle ]]; } && \
+ [[ "${MODULES[*]}" =~ \. ]]; }; then
+ MODULES=(.)
+ fi
+
+ # If the checkstyle configs change, check everything.
+ if [[ "${testtype}" == checkstyle ]] && [[ "${MODULES[*]}" =~
hbase-checkstyle ]]; then
+ MODULES=(.)
+ fi
+
+ if [[ ${testtype} == mvninstall ]]; then
+ # shellcheck disable=SC2086
+ personality_enqueue_module . ${extra}
+ return
+ fi
+
+ if [[ ${testtype} == spotbugs ]]; then
+ # Run spotbugs on each module individually to diff pre-patch and
post-patch results and
+ # report new warnings for changed modules only.
+ # For some reason, spotbugs on root is not working, but running on
individual modules is
+ # working. For time being, let it run on original list of CHANGED_MODULES.
HBASE-19491
+ for module in "${CHANGED_MODULES[@]}"; do
+ # skip spotbugs on any module that lacks content in `src/main/java`
+ if [[ "$(find "${BASEDIR}/${module}" -iname '*.java' -and -ipath
'*/src/main/java/*' \
+ -type f | wc -l | tr -d '[:space:]')" -eq 0 ]]; then
+ yetus_debug "no java files found under ${module}/src/main/java.
skipping."
+ continue
+ else
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ fi
+ done
+ return
+ fi
+
+ if [[ ${testtype} == compile ]] && [[ "${SKIP_ERRORPRONE}" != "true" ]] &&
+ [[ "${PATCH_BRANCH}" != branch-1* ]] ; then
+ extra="${extra} -PerrorProne"
+ fi
+
+ # If EXCLUDE_TESTS_URL/INCLUDE_TESTS_URL is set, fetches the url
+ # and sets -Dtest.exclude.pattern/-Dtest to exclude/include the
+ # tests respectively.
+ if [[ ${testtype} == unit ]]; then
+ local tests_arg=""
+ get_include_exclude_tests_arg tests_arg
+ #Phoenix traditially runs the full IT suite from Precommit
+ #keep it that way to ease transition
+ extra="verify ${extra} ${tests_arg}"
+
+ # Inject the jenkins build-id for our surefire invocations
+ # Used by zombie detection stuff, even though we're not including that yet.
+ if [ -n "${BUILD_ID}" ]; then
+ extra="${extra} -Dbuild.id=${BUILD_ID}"
+ fi
+
+ fi
+
+ for module in "${MODULES[@]}"; do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ done
+}
+
+## @description places where we override the built in assumptions about what
tests to run
+## @audience private
+## @stability evolving
+## @param filename of changed file
+function personality_file_tests
+{
+ local filename=$1
+ yetus_debug "Phoenix specific personality_file_tests"
+ # If we change checkstyle configs, run checkstyle
+ if [[ ${filename} =~ checkstyle.*\.xml ]]; then
+ yetus_debug "tests/checkstyle: ${filename}"
+ add_test checkstyle
+ fi
+ # fallback to checking which tests based on what yetus would do by default
+ if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
+ "${BUILDTOOL}_builtin_personality_file_tests" "${filename}"
+ elif declare -f builtin_personality_file_tests >/dev/null; then
+ builtin_personality_file_tests "${filename}"
+fi
+}
+
+## @description Uses relevant include/exclude env variable to fetch list of
included/excluded
+# tests and sets given variable to arguments to be passes to
maven command.
+## @audience private
+## @stability evolving
+## @param name of variable to set with maven arguments
+function get_include_exclude_tests_arg
+{
+ #Phoenix doesn't support this yet, but should
+ return
+ local __resultvar=$1
+ yetus_info "EXCLUDE_TESTS_URL=${EXCLUDE_TESTS_URL}"
+ yetus_info "INCLUDE_TESTS_URL=${INCLUDE_TESTS_URL}"
+ if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then
+ if wget "${EXCLUDE_TESTS_URL}" -O "excludes"; then
+ excludes=$(cat excludes)
+ yetus_debug "excludes=${excludes}"
+ if [[ -n "${excludes}" ]]; then
+ eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
+ fi
+ rm excludes
+ else
+ yetus_error "Wget error $? in fetching excludes file from url" \
+ "${EXCLUDE_TESTS_URL}. Ignoring and proceeding."
+ fi
+ elif [[ -n "$INCLUDE_TESTS_URL" ]]; then
+ if wget "$INCLUDE_TESTS_URL" -O "includes"; then
+ includes=$(cat includes)
+ yetus_debug "includes=${includes}"
+ if [[ -n "${includes}" ]]; then
+ eval "${__resultvar}='-Dtest=${includes}'"
+ fi
+ rm includes
+ else
+ yetus_error "Wget error $? in fetching includes file from url" \
+ "${INCLUDE_TESTS_URL}. Ignoring and proceeding."
+ fi
+ else
+ # Use branch specific exclude list when EXCLUDE_TESTS_URL and
INCLUDE_TESTS_URL are empty
+
FLAKY_URL="https://ci-hadoop.apache.org/job/HBase/job/HBase-Find-Flaky-Tests/job/${PATCH_BRANCH}/lastSuccessfulBuild/artifact/excludes/";
+ if wget "${FLAKY_URL}" -O "excludes"; then
+ excludes=$(cat excludes)
+ yetus_debug "excludes=${excludes}"
+ if [[ -n "${excludes}" ]]; then
+ eval "${__resultvar}='-Dtest.exclude.pattern=${excludes}'"
+ fi
+ rm excludes
+ else
+ yetus_error "Wget error $? in fetching excludes file from url" \
+ "${FLAKY_URL}. Ignoring and proceeding."
+ fi
+ fi
+}
+
+######################################
+
+add_test_type hbaseanti
+
+## @description hbaseanti file filter
+## @audience private
+## @stability evolving
+## @param filename
+function hbaseanti_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test hbaseanti
+ fi
+}
+
+## @description hbaseanti patch file check
+## @audience private
+## @stability evolving
+## @param filename
+function hbaseanti_patchfile
+{
+ local patchfile=$1
+ local warnings
+ local result
+
+ if [[ "${BUILDMODE}" = full ]]; then
+ return 0
+ fi
+
+ if ! verify_needed_test hbaseanti; then
+ return 0
+ fi
+
+ big_console_header "Checking for known anti-patterns"
+
+ start_clock
+
+ warnings=$(${GREP} -c 'new TreeMap<byte.*()' "${patchfile}")
+ if [[ ${warnings} -gt 0 ]]; then
+ add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern
where BYTES_COMPARATOR was omitted."
+ ((result=result+1))
+ fi
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+
+ add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
+ return 0
+}
+
+## @description process the javac output for generating WARNING/ERROR
+## @audience private
+## @stability evolving
+## @param input filename
+## @param output filename
+# Override the default javac_logfilter so that we can do a sort before
outputing the WARNING/ERROR.
+# This is because that the output order of the error prone warnings is not
stable, so the diff
+# method will report unexpected errors if we do not sort it. Notice that a
simple sort will cause
+# line number being sorted by lexicographical so the output maybe a bit
strange to human but it is
+# really hard to sort by file name first and then line number and column
number in shell...
+function hbase_javac_logfilter
+{
+ declare input=$1
+ declare output=$2
+
+ ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" | sort > "${output}"
+}
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index 549ecb0..809f745 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -33,45 +33,6 @@
<build>
<plugins>
- <!-- Add the ant-generated sources to the source path -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-site-plugin</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-module-markdown</artifactId>
- <version>1.3</version>
- </dependency>
- <dependency>
- <groupId>lt.velykis.maven.skins</groupId>
- <artifactId>reflow-velocity-tools</artifactId>
- <version>1.0.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.velocity</groupId>
- <artifactId>velocity</artifactId>
- <version>1.7</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <artifactId>exec-maven-plugin</artifactId>
- <groupId>org.codehaus.mojo</groupId>
- <version>1.2.1</version>
- <executions>
- <execution><!-- Run our version calculation script -->
- <id>Merge Language Reference</id>
- <phase>site</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>${basedir}/src/site/bin/merge.sh</executable>
- </configuration>
- </execution>
- </executions>
- </plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
@@ -527,17 +488,4 @@
<id>phoenix-hbase-compat-1.3.0</id>
</profile>
</profiles>
-
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </reporting>
</project>
diff --git a/phoenix-hbase-compat-1.3.0/pom.xml
b/phoenix-hbase-compat-1.3.0/pom.xml
index 1e4b575..7f86378 100644
--- a/phoenix-hbase-compat-1.3.0/pom.xml
+++ b/phoenix-hbase-compat-1.3.0/pom.xml
@@ -68,16 +68,4 @@
</dependency>
</dependencies>
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </reporting>
</project>
diff --git a/phoenix-hbase-compat-1.4.0/bin/pom.xml
b/phoenix-hbase-compat-1.4.0/bin/pom.xml
deleted file mode 100644
index e98c796..0000000
--- a/phoenix-hbase-compat-1.4.0/bin/pom.xml
+++ /dev/null
@@ -1,83 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0";
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation=
- "http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache</groupId>
- <artifactId>apache</artifactId>
- <version>21</version>
- </parent>
- <groupId>org.apache.phoenix</groupId>
- <artifactId>phoenix-hbase-compat-1.4.0</artifactId>
- <version>4.16.0-SNAPSHOT</version>
- <name>Phoenix Hbase 1.4.0 compatibility</name>
- <description>Compatibility module for HBase 1.4.0+</description>
-
- <build>
- <plugins>
- <!-- Setup eclipse -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-eclipse-plugin</artifactId>
- <version>2.9</version>
- <configuration>
- <buildcommands>
- <buildcommand>org.eclipse.jdt.core.javabuilder</buildcommand>
- </buildcommands>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.7</source>
- <target>1.7</target>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- <version>1.4.0</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </reporting>
-</project>
diff --git a/phoenix-hbase-compat-1.4.0/pom.xml
b/phoenix-hbase-compat-1.4.0/pom.xml
index e98c796..0faebdb 100644
--- a/phoenix-hbase-compat-1.4.0/pom.xml
+++ b/phoenix-hbase-compat-1.4.0/pom.xml
@@ -68,16 +68,4 @@
</dependency>
</dependencies>
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </reporting>
</project>
diff --git a/phoenix-hbase-compat-1.5.0/pom.xml
b/phoenix-hbase-compat-1.5.0/pom.xml
index e125270..c388190 100644
--- a/phoenix-hbase-compat-1.5.0/pom.xml
+++ b/phoenix-hbase-compat-1.5.0/pom.xml
@@ -69,16 +69,4 @@
</dependency>
</dependencies>
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </reporting>
</project>
diff --git a/pom.xml b/pom.xml
index 4376310..41c2197 100644
--- a/pom.xml
+++ b/pom.xml
@@ -137,6 +137,9 @@
<maven-eclipse-plugin.version>2.9</maven-eclipse-plugin.version>
<maven-build-helper-plugin.version>1.9.1</maven-build-helper-plugin.version>
<maven-enforcer-plugin.version>3.0.0-M3</maven-enforcer-plugin.version>
+
<maven-project-info-reports-plugin.version>3.1.1</maven-project-info-reports-plugin.version>
+ <spotbugs-maven-plugin.version>4.0.4</spotbugs-maven-plugin.version>
+ <spotbugs.version>4.1.2</spotbugs.version>
<jacoco-maven-plugin.version>0.8.5</jacoco-maven-plugin.version>
<!-- Plugin options -->
@@ -163,6 +166,11 @@
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>${maven-project-info-reports-plugin.version}</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>${maven-enforcer-plugin.version}</version>
</plugin>
@@ -174,6 +182,22 @@
<target>1.7</target>
</configuration>
</plugin>
+ <plugin>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ <version>${spotbugs-maven-plugin.version}</version>
+ <dependencies>
+ <dependency>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs</artifactId>
+ <version>${spotbugs.version}</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <effort>Max</effort>
+ <maxHeap>1024</maxHeap>
+ </configuration>
+ </plugin>
<!--This plugin's configuration is used to store Eclipse m2e settings
only. It has no influence on the Maven build itself. -->
<plugin>
@@ -505,10 +529,6 @@
</executions>
</plugin>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-site-plugin</artifactId>
- </plugin>
- <plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
@@ -1250,36 +1270,6 @@
</profile>
<profile>
- <id>owasp-dependency-check</id>
- <activation>
- <property>
- <name>owasp-check</name>
- </property>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.owasp</groupId>
- <artifactId>dependency-check-maven</artifactId>
- <version>${maven-owasp-plugin.version}</version>
- <configuration>
- <skipProvidedScope>true</skipProvidedScope>
- <skipRuntimeScope>true</skipRuntimeScope>
- <skipSystemScope>true</skipSystemScope>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>aggregate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
-
- <profile>
<id>codecoverage</id>
<activation>
<property>
@@ -1340,19 +1330,70 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>spotbugs-site</id>
+ <activation>
+ <property>
+ <name>!spotbugs.site</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ <configuration>
+ <spotbugsXmlOutput>true</spotbugsXmlOutput>
+ <xmlOutput>true</xmlOutput>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
<reporting>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <version>3.0.5</version>
- </plugin>
- </plugins>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.owasp</groupId>
+ <artifactId>dependency-check-maven</artifactId>
+ <version>${maven-owasp-plugin.version}</version>
+ <configuration>
+ <skipProvidedScope>true</skipProvidedScope>
+ <skipRuntimeScope>true</skipRuntimeScope>
+ <skipSystemScope>true</skipSystemScope>
+ </configuration>
+ <reportSets>
+ <reportSet>
+ <reports>
+ <report>aggregate</report>
+ </reports>
+ </reportSet>
+ </reportSets>
+ </plugin>
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <reportSets>
+ <reportSet>
+ <reports>
+ <report>report</report>
+ </reports>
+ </reportSet>
+ </reportSets>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ </plugin>
+ </plugins>
</reporting>
</project>
