This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/master by this push:
new 42222b9f4 TEZ-4624: Replace findbugs plugin with spotbugs (#410)
(Mahesh Raju Somalaraju reviewed by Ayush Saxena, Laszlo Bodor)
42222b9f4 is described below
commit 42222b9f47fd1a221fe5d7e6def69933a57159c6
Author: Mahesh Raju Somalaraju <[email protected]>
AuthorDate: Wed May 7 16:34:18 2025 +0530
TEZ-4624: Replace findbugs plugin with spotbugs (#410) (Mahesh Raju
Somalaraju reviewed by Ayush Saxena, Laszlo Bodor)
---
BUILDING.txt | 4 +-
Jenkinsfile | 7 +-
build-tools/docker/Dockerfile | 17 +--
build-tools/test-patch.sh | 68 ++++++------
pom.xml | 115 +++++++++++----------
tez-api/pom.xml | 4 -
.../java/org/apache/tez/client/TezClientUtils.java | 4 +-
tez-dag/pom.xml | 5 -
tez-runtime-internals/pom.xml | 4 -
tez-runtime-library/findbugs-exclude.xml | 24 +++++
10 files changed, 133 insertions(+), 119 deletions(-)
diff --git a/BUILDING.txt b/BUILDING.txt
index 57c9cd13c..16632dd84 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -8,7 +8,7 @@ Requirements:
* JDK 1.8+
* Maven 3.6.3 or later
-* Findbugs 2.0.2 or later (if running findbugs)
+* spotbugs 4.2.2 or later (if running spotbugs)
* ProtocolBuffer 3.21.1
* Internet connection for first build (to fetch all dependencies)
* Hadoop version should be 2.7.0 or higher.
@@ -36,7 +36,7 @@ Maven build goals:
* Compile : mvn compile
* Run tests : mvn test
* Create JAR : mvn package
- * Run findbugs : mvn compile findbugs:findbugs
+ * Run spotbugs : mvn compile spotbugs:spotbugs
* Run checkstyle : mvn compile checkstyle:checkstyle
* Install JAR in M2 cache : mvn install
* Deploy JAR to Maven repo : mvn deploy
diff --git a/Jenkinsfile b/Jenkinsfile
index 79c2a3e3c..d83e58918 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -116,11 +116,10 @@ pipeline {
YETUS_ARGS+=("--proclimit=5500")
YETUS_ARGS+=("--dockermemlimit=20g")
- # -1 findbugs issues that show up prior to the patch
being applied
- # YETUS_ARGS+=("--findbugs-strict-precheck")
-
+ # -1 spotbugs issues that show up prior to the patch
being applied
+ # YETUS_ARGS+=("--spotbugs-strict-precheck")
# rsync these files back into the archive dir
-
YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
+
YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")
# URL for user-side presentation in reports and such
to our artifacts
# (needs to match the archive bits below)
diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile
index 5b681067a..20c6a26a2 100644
--- a/build-tools/docker/Dockerfile
+++ b/build-tools/docker/Dockerfile
@@ -189,14 +189,15 @@ RUN java -version 2>&1 | grep "1.8.0" && rm -f
/usr/lib/jvm/default-java \
&& ln -s java-8-openjdk-amd64 /usr/lib/jvm/default-java
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-######
-# Install findbugs
-######
-# hadolint ignore=DL3008
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y
findbugs \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-ENV FINDBUGS_HOME /usr
+#######
+# Install SpotBugs 4.2.2
+#######
+RUN mkdir -p /opt/spotbugs \
+ && curl -L -s -S
https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz
\
+ -o /opt/spotbugs.tgz \
+ && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
+ && chmod +x /opt/spotbugs/bin/*
+ENV SPOTBUGS_HOME /opt/spotbugs
######
# Install maven
diff --git a/build-tools/test-patch.sh b/build-tools/test-patch.sh
index 9b457af22..feb964489 100755
--- a/build-tools/test-patch.sh
+++ b/build-tools/test-patch.sh
@@ -40,7 +40,7 @@ DIFF=${DIFF:-diff}
JIRACLI=${JIRA:-jira}
SED=${SED:-sed}
CURL=${CURL:-curl}
-FINDBUGS_HOME=${FINDBUGS_HOME}
+SPOTBUGS_HOME=${SPOTBUGS_HOME}
###############################################################################
printUsage() {
@@ -60,7 +60,7 @@ printUsage() {
echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')"
echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')"
echo "--diff-cmd=<cmd> The 'diff' command to use (default 'diff')"
- echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME
environment variable)"
+ echo "--spotbugs-home=<path> Spotbugs home directory (default SPOTBUGS_HOME
environment variable)"
echo "--dirty-workspace Allow the local git workspace to have
uncommitted changes"
echo "--run-tests Run all tests below the base directory"
echo
@@ -115,8 +115,8 @@ parseArgs() {
--jira-password=*)
JIRA_PASSWD=${i#*=}
;;
- --findbugs-home=*)
- FINDBUGS_HOME=${i#*=}
+ --spotbugs-home=*)
+ SPOTBUGS_HOME=${i#*=}
;;
--dirty-workspace)
DIRTY_WORKSPACE=true
@@ -507,7 +507,7 @@ $JIRA_COMMENT_FOOTER"
###############################################################################
-### Install the new jars so tests and findbugs can find all of the updated jars
+### Install the new jars so tests and spotbugs can find all of the updated jars
buildAndInstall () {
echo ""
echo ""
@@ -525,67 +525,67 @@ buildAndInstall () {
###############################################################################
-### Check there are no changes in the number of Findbugs warnings
-checkFindbugsWarnings () {
+### Check there are no changes in the number of Spotbugs warnings
+checkSpotbugsWarnings () {
echo ""
echo ""
echo "======================================================================"
echo "======================================================================"
- echo " Determining number of patched Findbugs warnings."
+ echo " Determining number of patched Spotbugs warnings."
echo "======================================================================"
echo "======================================================================"
echo ""
echo ""
rc=0
- echo " Running findbugs "
- echo "$MVN clean test findbugs:findbugs -DskipTests < /dev/null >
$PATCH_DIR/patchFindBugsOutput.txt 2>&1"
- $MVN clean test findbugs:findbugs -DskipTests < /dev/null >
$PATCH_DIR/patchFindBugsOutput.txt 2>&1
+ echo " Running spotbugs "
+ echo "$MVN clean test spotbugs:spotbugs -DskipTests < /dev/null >
$PATCH_DIR/patchSpotBugsOutput.txt 2>&1"
+ $MVN clean test spotbugs:spotbugs -DskipTests < /dev/null >
$PATCH_DIR/patchSpotBugsOutput.txt 2>&1
rc=$?
- findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/)
{ print substr($0, RSTART + 22, RLENGTH - 31); exit }'
"${PATCH_DIR}/patchFindBugsOutput.txt")
+ spotbugs_version=$(${AWK} 'match($0, /spotbugs-maven-plugin:[^:]*:spotbugs/)
{ print substr($0, RSTART + 22, RLENGTH - 31); exit }'
"${PATCH_DIR}/patchSpotBugsOutput.txt")
if [ $rc != 0 ] ; then
JIRA_COMMENT="$JIRA_COMMENT
- {color:red}-1 findbugs{color}. The patch appears to cause Findbugs
(version ${findbugs_version}) to fail."
+ {color:red}-1 spotbugs{color}. The patch appears to cause Spotbugs
(version ${spotbugs_version}) to fail."
return 1
fi
- findbugsWarnings=0
- for file in $(find $BASEDIR -name findbugsXml.xml)
+ spotbugsWarnings=0
+ for file in $(find $BASEDIR -name spotbugsXml.xml)
do
relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix
- if [ ! $relative_file == "target/findbugsXml.xml" ]; then
- module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing
path
+ if [ ! $relative_file == "target/spotbugsXml.xml" ]; then
+ module_suffix=${relative_file%/target/spotbugsXml.xml} # strip trailing
path
module_suffix=`basename ${module_suffix}`
fi
- cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
- $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \
- $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
- $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
- newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000"
$PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
- $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print
$1}'`
- echo "Found $newFindbugsWarnings Findbugs warnings ($file)"
- findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings))
- $FINDBUGS_HOME/bin/convertXmlToText -html \
- $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \
- $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
- if [[ $newFindbugsWarnings > 0 ]] ; then
- JIRA_COMMENT_FOOTER="Findbugs warnings:
$BUILD_URL/artifact/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
+ cp $file $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml
+ $SPOTBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \
+ $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \
+ $PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml
+ newSpotbugsWarnings=`$SPOTBUGS_HOME/bin/filterBugs -first "01/01/2000"
$PATCH_DIR/patchSpotbugsWarnings${module_suffix}.xml \
+ $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml | $AWK '{print
$1}'`
+ echo "Found $newSpotbugsWarnings Spotbugs warnings ($file)"
+ spotbugsWarnings=$((spotbugsWarnings+newSpotbugsWarnings))
+ $SPOTBUGS_HOME/bin/convertXmlToText -html \
+ $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.xml \
+ $PATCH_DIR/newPatchSpotbugsWarnings${module_suffix}.html
+ if [[ $newSpotbugsWarnings > 0 ]] ; then
+ JIRA_COMMENT_FOOTER="Spotbugs warnings:
$BUILD_URL/artifact/patchprocess/newPatchSpotbugsWarnings${module_suffix}.html
$JIRA_COMMENT_FOOTER"
fi
done
- if [[ $findbugsWarnings -gt 0 ]] ; then
+ if [[ $spotbugsWarnings -gt 0 ]] ; then
JIRA_COMMENT="$JIRA_COMMENT
- {color:red}-1 findbugs{color}. The patch appears to introduce
$findbugsWarnings new Findbugs (version ${findbugs_version}) warnings."
+ {color:red}-1 spotbugs{color}. The patch appears to introduce
$spotbugsWarnings new Spotbugs (version ${spotbugs_version}) warnings."
return 1
fi
JIRA_COMMENT="$JIRA_COMMENT
- {color:green}+1 findbugs{color}. The patch does not introduce any new
Findbugs (version ${findbugs_version}) warnings."
+ {color:green}+1 spotbugs{color}. The patch does not introduce any new
Spotbugs (version ${spotbugs_version}) warnings."
return 0
}
@@ -805,7 +805,7 @@ fi
checkJavadocWarnings
(( RESULT = RESULT + $? ))
buildAndInstall
-checkFindbugsWarnings
+checkSpotbugsWarnings
(( RESULT = RESULT + $? ))
checkReleaseAuditWarnings
(( RESULT = RESULT + $? ))
diff --git a/pom.xml b/pom.xml
index 0ab151d9d..4deb501e3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -70,7 +70,8 @@
<commons-lang.version>2.6</commons-lang.version>
<clover.license>${user.home}/clover.license</clover.license>
<dependency-check-maven.version>3.2.0</dependency-check-maven.version>
- <findbugs-maven-plugin.version>3.0.5</findbugs-maven-plugin.version>
+ <spotbugs.version>4.2.2</spotbugs.version>
+ <spotbugs-maven-plugin.version>4.2.0</spotbugs-maven-plugin.version>
<frontend-maven-plugin.version>1.8.0</frontend-maven-plugin.version>
<guava.version>32.0.1-jre</guava.version>
<hadoop.version>3.4.1</hadoop.version>
@@ -770,11 +771,6 @@
<artifactId>jettison</artifactId>
<version>${jettison.version}</version>
</dependency>
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- <version>${jsr305.version}</version>
- </dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
@@ -970,9 +966,20 @@
<version>${dependency-check-maven.version}</version>
</plugin>
<plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <version>${findbugs-maven-plugin.version}</version>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ <version>${spotbugs-maven-plugin.version}</version>
+ <dependencies>
+ <dependency>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs</artifactId>
+ <version>${spotbugs.version}</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+
<excludeFilterFile>${basedir}/findbugs-exclude.xml</excludeFilterFile>
+ <failOnError>false</failOnError>
+ </configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
@@ -1119,10 +1126,11 @@
<artifactId>build-helper-maven-plugin</artifactId>
</plugin>
<plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
<configuration>
<excludeFilterFile>${basedir}/findbugs-exclude.xml</excludeFilterFile>
+ <failOnError>false</failOnError>
</configuration>
</plugin>
<plugin>
@@ -1214,51 +1222,6 @@
</plugins>
</build>
</profile>
- <profile>
- <id>findbugs</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- <property>
- <name>findbugs</name>
- </property>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>site</id>
- <phase>pre-site</phase>
- <goals>
- <goal>findbugs</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
-
<excludeFilterFile>${basedir}/findbugs-exclude.xml</excludeFilterFile>
- </configuration>
- </plugin>
- </plugins>
- </build>
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>findbugs-maven-plugin</artifactId>
- <configuration>
- <formats>
- <format>xml</format>
- <format>html</format>
- </formats>
- <canGenerate>true</canGenerate>
-
<excludeFilterFile>${basedir}/findbugs-exclude.xml</excludeFilterFile>
- </configuration>
- </plugin>
- </plugins>
- </reporting>
- </profile>
<profile>
<id>sources</id>
<build>
@@ -1383,6 +1346,46 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>spotbugs</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ <property>
+ <name>spotbugs</name>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ <version>${spotbugs-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>spotbugs</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+
<excludeFilterFile>${basedir}/findbugs-exclude.xml</excludeFilterFile>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ <configuration>
+
<excludeFilterFile>${basedir}/findbugs-exclude.xml</excludeFilterFile>
+ </configuration>
+ </plugin>
+ </plugins>
+ </reporting>
+ </profile>
</profiles>
<!-- TODO: TEZ-4597: IncludePublicAnnotationsStandardDoclet is not JDK9+
compatible -->
diff --git a/tez-api/pom.xml b/tez-api/pom.xml
index 4e6c27e18..08eaa8054 100644
--- a/tez-api/pom.xml
+++ b/tez-api/pom.xml
@@ -85,10 +85,6 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- </dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index bae0174b3..eff4d4bef 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -1006,11 +1006,11 @@ public final class TezClientUtils {
return javaOpts;
}
- if ((maxHeapFactor <= 0 && Double.valueOf("-1") != maxHeapFactor) ||
maxHeapFactor >= 1) {
+ if ((maxHeapFactor <= 0 && Double.parseDouble("-1") != maxHeapFactor) ||
maxHeapFactor >= 1) {
return javaOpts;
}
- if (Double.valueOf("-1") == maxHeapFactor) {
+ if (Double.parseDouble("-1") == maxHeapFactor) {
maxHeapFactor = resource.getMemory() <
TezConstants.TEZ_CONTAINER_SMALL_SLAB_BOUND_MB
? TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_SMALL_SLAB
: TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_LARGE_SLAB;
diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml
index 779f7ed82..93ab9c51b 100644
--- a/tez-dag/pom.xml
+++ b/tez-dag/pom.xml
@@ -111,11 +111,6 @@
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
diff --git a/tez-runtime-internals/pom.xml b/tez-runtime-internals/pom.xml
index e235e933c..d95990c6d 100644
--- a/tez-runtime-internals/pom.xml
+++ b/tez-runtime-internals/pom.xml
@@ -86,10 +86,6 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
diff --git a/tez-runtime-library/findbugs-exclude.xml
b/tez-runtime-library/findbugs-exclude.xml
index caa46c552..7ac769fca 100644
--- a/tez-runtime-library/findbugs-exclude.xml
+++ b/tez-runtime-library/findbugs-exclude.xml
@@ -220,4 +220,28 @@
<Field name="shuffleSchedulerThread"/>
<Bug pattern="IS2_INCONSISTENT_SYNC"/>
</Match>
+
+ <Match>
+ <Class name="org.apache.tez.runtime.library.api.TezRuntimeConfiguration"/>
+ <Bug pattern="DLS_DEAD_LOCAL_STORE"/>
+ </Match>
+
+ <Match>
+ <Class
name="org.apache.tez.runtime.library.common.shuffle.impl.ShuffleManager"/>
+ <Field name="localDisks"/>
+ <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"/>
+ </Match>
+
+ <Match>
+ <Class
name="org.apache.tez.runtime.library.cartesianproduct.CartesianProductCombination"/>
+ <Method name="nextTask" params="" returns="boolean"/>
+ <Bug pattern="DLS_DEAD_LOCAL_STORE"/>
+ </Match>
+
+ <Match>
+ <Class
name="org.apache.tez.runtime.library.cartesianproduct.CartesianProductCombination"/>
+ <Method name="nextTaskWithFixedChunk" params="" returns="boolean"/>
+ <Bug pattern="DLS_DEAD_LOCAL_STORE"/>
+ </Match>
+
</FindBugsFilter>
