This is an automated email from the ASF dual-hosted git repository.
nihaljain pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new aad1242a7ae HBASE-28433 Modify the assembly to not include test jars
and their transitive dependencies (#6184) (#6452)
aad1242a7ae is described below
commit aad1242a7aebd066ec25f1fe08e75608cc04fe38
Author: Nihal Jain <[email protected]>
AuthorDate: Mon Nov 11 11:11:23 2024 +0530
HBASE-28433 Modify the assembly to not include test jars and their
transitive dependencies (#6184) (#6452)
- Before this change, our assembly included test jars and test
dependencies, which added unnecessary bloat to the assembly and classpath. This
not only increases the distribution size but also potentially introduces
CVE-prone JARs into the binary assemblies.
- With this change, we modify the assembly to exclude these test jars and
their dependencies. This will result in a leaner, more secure assembly with a
faster startup time.
- To ensure cached_classpath is same as before this change, we retain some
jars (including test jars) in assembly pom.xml which we actually remove during
tarball creation.
Signed-off-by: Istvan Toth <[email protected]>
(cherry picked from commit 3fbe4fbb68e33f6ed796d268961ae05e19307f65)
---
bin/hbase | 10 +---
hbase-assembly/pom.xml | 14 +++++
.../src/main/assembly/client-components.xml | 55 -----------------
hbase-assembly/src/main/assembly/client.xml | 11 +---
hbase-assembly/src/main/assembly/components.xml | 70 ----------------------
.../src/main/assembly/hadoop-three-compat.xml | 15 +----
hbase-endpoint/pom.xml | 1 +
hbase-examples/pom.xml | 1 +
hbase-it/pom.xml | 2 +
hbase-shell/pom.xml | 2 +
hbase-thrift/pom.xml | 1 +
pom.xml | 11 ----
12 files changed, 27 insertions(+), 166 deletions(-)
diff --git a/bin/hbase b/bin/hbase
index 937f7c22f2d..68d85d6805f 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -514,8 +514,9 @@ add_opentelemetry_agent() {
#Add the development env class path stuff
if $in_dev_env; then
+ # The below line makes available all the jars required in dev_env, even if
we may not add them
+ # to assembly. See HBASE-28433 for more info.
add_maven_deps_to_classpath "cached_classpath.txt"
-
if [[ $jline_needed ]]; then
add_maven_deps_to_classpath "cached_classpath_jline.txt"
elif [[ $jruby_needed ]]; then
@@ -766,13 +767,6 @@ elif [ "$COMMAND" = "copyreppeers" ] ; then
CLASS='org.apache.hadoop.hbase.replication.CopyReplicationPeers'
else
CLASS=$COMMAND
-if [[ "$CLASS" =~ .*IntegrationTest.* ]] ; then
- for f in ${HBASE_HOME}/lib/test/*.jar; do
- if [ -f "${f}" ]; then
- CLASSPATH="${CLASSPATH}:${f}"
- fi
- done
- fi
fi
add_jdk17_jvm_flags
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
index 5b51b88f931..90be45e9cbf 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly/pom.xml
@@ -48,6 +48,8 @@
<artifactId>hbase-shaded-mapreduce</artifactId>
</dependency>
<!-- Intra-project dependencies -->
+ <!-- We don't really add this to assembly tarball, we retain it here just
to dump it into
+ cached_classpath.txt ! See HBASE-28433 for more info. -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-it</artifactId>
@@ -55,6 +57,8 @@
</dependency>
<!-- Hamcrest is required by hbase-it (via junit), but as long as we're
grabbing the hbase-it test-jar,
maven dependency resolution won't pick it up for us. -->
+ <!-- We don't really add this to assembly tarball, we retain it here just
to dump it into
+ cached_classpath.txt ! See HBASE-28433 for more info. -->
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
@@ -73,6 +77,8 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-mapreduce</artifactId>
</dependency>
+ <!-- We don't really add this to assembly tarball, we retain it here just
to dump it into
+ cached_classpath.txt ! See HBASE-28433 for more info. -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-mapreduce</artifactId>
@@ -83,6 +89,8 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-procedure</artifactId>
</dependency>
+ <!-- We don't really add this to assembly tarball, we retain it here just
to dump it into
+ cached_classpath.txt ! See HBASE-28433 for more info. -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-procedure</artifactId>
@@ -115,6 +123,8 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-external-blockcache</artifactId>
</dependency>
+ <!-- We don't really add this to assembly tarball, we retain it here just
to dump it into
+ cached_classpath.txt ! See HBASE-28433 for more info. -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-testing-util</artifactId>
@@ -235,10 +245,14 @@
<groupId>io.opentelemetry.javaagent</groupId>
<artifactId>opentelemetry-javaagent</artifactId>
</dependency>
+ <!-- We don't really add this to assembly tarball, we retain it here just
to dump it into
+ cached_classpath.txt ! See HBASE-28433 for more info. -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
+ <!-- We don't really add this to assembly tarball, we retain it here just
to dump it into
+ cached_classpath.txt ! See HBASE-28433 for more info. -->
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
diff --git a/hbase-assembly/src/main/assembly/client-components.xml
b/hbase-assembly/src/main/assembly/client-components.xml
index 7cb97dd6822..b0e359485a3 100644
--- a/hbase-assembly/src/main/assembly/client-components.xml
+++ b/hbase-assembly/src/main/assembly/client-components.xml
@@ -87,60 +87,5 @@
<include>*.dylib</include>
</includes>
</fileSet>
- <!-- This is only necessary until maven fixes the intra-project dependency
bug
- in maven 3.0. Until then, we have to include the test jars for
sub-projects. When
- fixed, the below dependencySet stuff is sufficient for pulling in the
test jars as
- well, as long as they are added as dependencies in this project. Right
now, we only
- have 1 submodule to accumulate, but we can copy/paste as necessary until
maven is
- fixed. -->
- <!-- Used by PE and ltt -->
- <fileSet>
- <directory>${project.basedir}/../hbase-server/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${server.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/../hbase-mapreduce/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${mapreduce.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/../hbase-common/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${common.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/../hbase-asyncfs/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${asyncfs.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/../hbase-zookeeper/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${zookeeper.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/../hbase-procedure/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${procedure.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
</fileSets>
</component>
diff --git a/hbase-assembly/src/main/assembly/client.xml
b/hbase-assembly/src/main/assembly/client.xml
index cd627ae9589..9f95e139332 100644
--- a/hbase-assembly/src/main/assembly/client.xml
+++ b/hbase-assembly/src/main/assembly/client.xml
@@ -45,6 +45,8 @@
<dependencySets>
<dependencySet>
<excludes>
+ <exclude>org.apache.hadoop:*:test-jar</exclude>
+ <exclude>org.apache.hbase:*:test-jar</exclude>
<!-- Exclude J2EE libraries that get pulled in when building on
JDK11 -->
<exclude>com.sun.xml.ws:jaxws-ri</exclude>
<!-- Exclude libraries that we put in their own dirs under lib/ -->
@@ -167,15 +169,6 @@
<include>io.opentelemetry.javaagent:*</include>
</includes>
</dependencySet>
- <!-- Adds junit libs to lib/test -->
- <dependencySet>
- <outputDirectory>lib/test</outputDirectory>
- <includes>
- <include>junit:junit</include>
- <include>org.hamcrest:hamcrest-core</include>
- <include>org.mockito:mockito-core</include>
- </includes>
- </dependencySet>
</dependencySets>
</assembly>
diff --git a/hbase-assembly/src/main/assembly/components.xml
b/hbase-assembly/src/main/assembly/components.xml
index 3e1394e7d5b..f79b2e754ca 100644
--- a/hbase-assembly/src/main/assembly/components.xml
+++ b/hbase-assembly/src/main/assembly/components.xml
@@ -109,75 +109,5 @@
<include>**/*.h</include>
</includes>
</fileSet>
- <!-- This is only necessary until maven fixes the intra-project dependency
bug
- in maven 3.0. Until then, we have to include the test jars for
sub-projects. When
- fixed, the below dependencySet stuff is sufficient for pulling in the
test jars as
- well, as long as they are added as dependencies in this project. Right
now, we only
- have 1 submodule to accumulate, but we can copy/paste as necessary until
maven is
- fixed. -->
- <fileSet>
- <directory>${project.basedir}/../hbase-server/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${server.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/../hbase-it/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${it.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/../hbase-annotations/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${annotations.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/../hbase-common/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${common.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/../hbase-mapreduce/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${mapreduce.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/../hbase-asyncfs/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${asyncfs.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/../hbase-zookeeper/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${zookeeper.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>${project.basedir}/../hbase-procedure/target/</directory>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>${procedure.test.jar}</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
</fileSets>
</component>
diff --git a/hbase-assembly/src/main/assembly/hadoop-three-compat.xml
b/hbase-assembly/src/main/assembly/hadoop-three-compat.xml
index 4ba995409f0..f5da20b2131 100644
--- a/hbase-assembly/src/main/assembly/hadoop-three-compat.xml
+++ b/hbase-assembly/src/main/assembly/hadoop-three-compat.xml
@@ -33,7 +33,6 @@
<useAllReactorProjects>true</useAllReactorProjects>
<includes>
<!-- Keep this list sorted by name -->
- <include>org.apache.hbase:hbase-annotations</include>
<include>org.apache.hbase:hbase-asyncfs</include>
<include>org.apache.hbase:hbase-backup</include>
<include>org.apache.hbase:hbase-balancer</include>
@@ -43,7 +42,6 @@
<include>org.apache.hbase:hbase-external-blockcache</include>
<include>org.apache.hbase:hbase-hadoop-compat</include>
<include>org.apache.hbase:hbase-http</include>
- <include>org.apache.hbase:hbase-it</include>
<include>org.apache.hbase:hbase-logging</include>
<include>org.apache.hbase:hbase-mapreduce</include>
<include>org.apache.hbase:hbase-diagnostics</include>
@@ -55,7 +53,6 @@
<include>org.apache.hbase:hbase-rest</include>
<include>org.apache.hbase:hbase-server</include>
<include>org.apache.hbase:hbase-shell</include>
- <include>org.apache.hbase:hbase-testing-util</include>
<include>org.apache.hbase:hbase-thrift</include>
<include>org.apache.hbase:hbase-zookeeper</include>
</includes>
@@ -66,6 +63,8 @@
<dependencySets>
<dependencySet>
<excludes>
+ <exclude>org.apache.hadoop:*:test-jar</exclude>
+ <exclude>org.apache.hbase:*:test-jar</exclude>
<!-- Exclude pom file -->
<exclude>com.sun.xml.ws:jaxws-ri:pom</exclude>
<!-- Exclude libraries that we put in their own dirs under lib/
-->
@@ -206,16 +205,6 @@
<include>io.opentelemetry.javaagent:*</include>
</includes>
</dependencySet>
-
- <!-- Adds junit libs to lib/test -->
- <dependencySet>
- <outputDirectory>lib/test</outputDirectory>
- <includes>
- <include>junit:junit</include>
- <include>org.hamcrest:hamcrest-core</include>
- <include>org.mockito:mockito-core</include>
- </includes>
- </dependencySet>
</dependencySets>
</assembly>
diff --git a/hbase-endpoint/pom.xml b/hbase-endpoint/pom.xml
index 4b3e54418b5..e6859730041 100644
--- a/hbase-endpoint/pom.xml
+++ b/hbase-endpoint/pom.xml
@@ -256,6 +256,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
diff --git a/hbase-examples/pom.xml b/hbase-examples/pom.xml
index c39e977cba6..816e08ad32a 100644
--- a/hbase-examples/pom.xml
+++ b/hbase-examples/pom.xml
@@ -310,6 +310,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>javax.xml.bind.</groupId>
diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml
index 6b526a55202..a024a848ad6 100644
--- a/hbase-it/pom.xml
+++ b/hbase-it/pom.xml
@@ -370,6 +370,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>javax.xml.bind</groupId>
@@ -384,6 +385,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
+ <scope>test</scope>
</dependency>
</dependencies>
<build>
diff --git a/hbase-shell/pom.xml b/hbase-shell/pom.xml
index 06caf29cbe5..3cd7b5fb482 100644
--- a/hbase-shell/pom.xml
+++ b/hbase-shell/pom.xml
@@ -190,6 +190,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
@@ -204,6 +205,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
+ <scope>test</scope>
</dependency>
</dependencies>
<build>
diff --git a/hbase-thrift/pom.xml b/hbase-thrift/pom.xml
index b0fcd8e3d43..4f6ccfd18be 100644
--- a/hbase-thrift/pom.xml
+++ b/hbase-thrift/pom.xml
@@ -441,6 +441,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
diff --git a/pom.xml b/pom.xml
index e250145074a..d5a1bb824f3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -918,17 +918,6 @@
<jacoco.version>0.8.8</jacoco.version>
<jacocoArgLine/>
<sonar-maven-plugin.version>3.9.1.2184</sonar-maven-plugin.version>
- <!-- Intraproject jar naming properties -->
- <!-- TODO this is pretty ugly, but works for the moment.
- Modules are pretty heavy-weight things, so doing this work isn't too
bad. -->
-
<server.test.jar>hbase-server-${project.version}-tests.jar</server.test.jar>
-
<common.test.jar>hbase-common-${project.version}-tests.jar</common.test.jar>
-
<procedure.test.jar>hbase-procedure-${project.version}-tests.jar</procedure.test.jar>
- <it.test.jar>hbase-it-${project.version}-tests.jar</it.test.jar>
-
<annotations.test.jar>hbase-annotations-${project.version}-tests.jar</annotations.test.jar>
-
<mapreduce.test.jar>hbase-mapreduce-${project.version}-tests.jar</mapreduce.test.jar>
-
<zookeeper.test.jar>hbase-zookeeper-${project.version}-tests.jar</zookeeper.test.jar>
-
<asyncfs.test.jar>hbase-asyncfs-${project.version}-tests.jar</asyncfs.test.jar>
<shell-executable>bash</shell-executable>
<surefire.provider>surefire-junit47</surefire.provider>
<!-- default: run small & medium, medium with 2 threads -->
