This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 3875fff5e11 HBASE-28434 Update assembly to create a tarball with
hadoop and without hadoop (#5766)
3875fff5e11 is described below
commit 3875fff5e1179d7a37e93b9a72435a2132b2559b
Author: Istvan Toth <[email protected]>
AuthorDate: Mon Jan 27 07:03:23 2025 +0100
HBASE-28434 Update assembly to create a tarball with hadoop and without
hadoop (#5766)
Signed-off-by: Duo Zhang <[email protected]>
(cherry picked from commit 93b8d7bb7740246234578e701535ace33e30e556)
---
bin/hbase | 8 ++
.../pom.xml | 116 +++++++++++++--
.../src/main/assembly/components.xml | 113 +++++++++++++++
.../src/main/assembly/hadoop-three-compat.xml | 158 +++++++++++++++++++++
.../src/main/resources/META-INF/LEGAL | 5 +
hbase-assembly/pom.xml | 1 +
pom.xml | 6 +
7 files changed, 399 insertions(+), 8 deletions(-)
diff --git a/bin/hbase b/bin/hbase
index c562ad8fb3d..8f58bf578a5 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -221,6 +221,14 @@ if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ]
; then
HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which
hadoop 2>/dev/null)
fi
+#If Hadoop is not specified with HADOOP_HOME, check that the assembly includes
Hadoop
+if [[ -z "${HADOOP_IN_PATH}" && ! -e "lib/hadoop-common*" ]] ; then
+ echo Installation does not contain Hadoop, and HADOOP_HOME does not point to
a Hadoop installation.
+ echo Specify a compatible Hadoop installation via HADOOP_HOME, or use the
HBase assembly variant
+ echo that includes Hadoop.
+ exit 1
+fi
+
# Add libs to CLASSPATH
declare shaded_jar
diff --git a/hbase-assembly/pom.xml b/hbase-assembly-byo-hadoop/pom.xml
similarity index 73%
copy from hbase-assembly/pom.xml
copy to hbase-assembly-byo-hadoop/pom.xml
index c9db9a9ddee..30fd102ff45 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly-byo-hadoop/pom.xml
@@ -26,19 +26,112 @@
<version>${revision}</version>
<relativePath>../hbase-build-configuration</relativePath>
</parent>
- <artifactId>hbase-assembly</artifactId>
+ <artifactId>hbase-assembly-byo-hadoop</artifactId>
<packaging>pom</packaging>
- <name>Apache HBase - Assembly</name>
- <description>Module that does project assembly and that is all that it
does.</description>
+ <name>Apache HBase - Assembly without Hadoop and its transitive
dependencies</name>
+ <description>Module that does project assembly without Hadoop and its
transitive dependencies and that is all that it does.</description>
<properties>
<license.bundles.dependencies>true</license.bundles.dependencies>
+ <!-- So that we don't have to add redundant exclusions to provided scope
dependencies -->
+ <enforcer.skip>true</enforcer.skip>
</properties>
<dependencies>
- <!-- client artifacts for downstream use -->
+ <!-- Set every Hadoop dependency as provided. The byo-hadoop assembly
relies an external Hadoop installation -->
+ <!-- These MUST come first, otherwise some transitive dependencies are not
be excluded -->
+ <!-- When Hadoop and HBase depend on different versions of the same
artifact, then even if it is provided,
+ the first version specified is used, so in some cases we need to exclude
those transitive dependencies from
+ Hadoop to have an equivalent effective classpath to the assembly with
Hadoop. -->
+ <!-- The versions may be marked as redundant by some IDEs, but they are
not defined elsewhere for the provided scope -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${hadoop-three.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-shaded-client</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <version>${hadoop-three.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hadoop-three.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-jaxb-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>jakarta.xml.bind</groupId>
+ <artifactId>jakarta.xml.bind-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop-three.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <version>${hadoop-three.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop-three.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <version>${hadoop-three.version}</version>
+ <scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop-three.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.checkerframework</groupId>
+ <artifactId>checker-qual</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop-three.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <version>${hadoop-three.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <version>${hadoop-three.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop-three.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <!-- client artifacts for downstream use -->
+ <!-- skip hbase-shaded-client. We don't want any Hadoop code in the
assembly-->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-client-byo-hadoop</artifactId>
@@ -47,6 +140,7 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-mapreduce</artifactId>
</dependency>
+ <!-- Intra-project dependencies -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-openssl</artifactId>
@@ -141,6 +235,13 @@
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-hbtop</artifactId>
+ <exclusions>
+ <!-- hbase-top will pick up hbase-shaded-client-byo-hadoop from the
classpath -->
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
@@ -275,13 +376,12 @@
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<!--Else will use hbase-assembly as final name.-->
- <finalName>hbase-${project.version}</finalName>
+ <finalName>hbase-byo-hadoop-${project.version}</finalName>
<skipAssembly>false</skipAssembly>
<appendAssemblyId>true</appendAssemblyId>
<tarLongFileMode>posix</tarLongFileMode>
<descriptors>
<descriptor>${assembly.file}</descriptor>
- <descriptor>src/main/assembly/client.xml</descriptor>
</descriptors>
</configuration>
</plugin>
diff --git a/hbase-assembly-byo-hadoop/src/main/assembly/components.xml
b/hbase-assembly-byo-hadoop/src/main/assembly/components.xml
new file mode 100644
index 00000000000..f79b2e754ca
--- /dev/null
+++ b/hbase-assembly-byo-hadoop/src/main/assembly/components.xml
@@ -0,0 +1,113 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<!-- Filesets shared by different binary tars. -->
+<component>
+ <fileSets>
+ <!--Copy over the site if built as docs dir -->
+ <fileSet>
+ <directory>${project.basedir}/../target/site</directory>
+ <outputDirectory>docs</outputDirectory>
+ <!--Don't include dev apidocs. They bloat the tgz by almost 50%.
+ See HBASE-20149-->
+ <excludes>
+ <exclude>**/*devapidocs/**</exclude>
+ </excludes>
+ </fileSet>
+ <!-- Include top level text files-->
+ <fileSet>
+ <directory>${project.basedir}/..</directory>
+ <outputDirectory>.</outputDirectory>
+ <includes>
+ <include>CHANGES.md</include>
+ <include>RELEASENOTES.md</include>
+ <include>README.txt</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <!-- Include the top level conf directory -->
+ <fileSet>
+ <directory>${project.basedir}/../conf</directory>
+ <outputDirectory>conf</outputDirectory>
+ <fileMode>0644</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+ <!-- Include top level bin directory -->
+ <!-- First copy all but the *.cmd files-->
+ <fileSet>
+ <directory>${project.basedir}/../bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <excludes>
+ <exclude>**/*.cmd</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+ <!--Now do the cmd files; we do not want these executable.-->
+ <fileSet>
+ <directory>${project.basedir}/../bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <includes>
+ <include>**/*.cmd</include>
+ </includes>
+ </fileSet>
+ <!-- Move the webapps to the webapp dir -->
+ <fileSet>
+
<directory>${project.basedir}/../hbase-server/target/hbase-webapps</directory>
+ <outputDirectory>hbase-webapps</outputDirectory>
+ <fileMode>0644</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+ <fileSet>
+
<directory>${project.basedir}/../hbase-thrift/target/hbase-webapps</directory>
+ <outputDirectory>hbase-webapps/</outputDirectory>
+ <fileMode>0644</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+ <fileSet>
+
<directory>${project.basedir}/../hbase-rest/target/hbase-webapps</directory>
+ <outputDirectory>hbase-webapps/</outputDirectory>
+ <fileMode>0644</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+ <!-- Include native libraries -->
+ <fileSet>
+ <directory>${project.basedir}/../hbase-server/target/native</directory>
+ <outputDirectory>lib/native</outputDirectory>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ <includes>
+ <include>*.so</include>
+ <include>*.dylib</include>
+ </includes>
+ </fileSet>
+ <!-- includes for native client -->
+ <fileSet>
+
<directory>${project.basedir}/../hbase-common/target/generated-sources/native</directory>
+ <outputDirectory>include</outputDirectory>
+ <fileMode>0644</fileMode>
+ <directoryMode>0755</directoryMode>
+ <includes>
+ <include>**/*.h</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+</component>
diff --git
a/hbase-assembly-byo-hadoop/src/main/assembly/hadoop-three-compat.xml
b/hbase-assembly-byo-hadoop/src/main/assembly/hadoop-three-compat.xml
new file mode 100644
index 00000000000..794d1db8517
--- /dev/null
+++ b/hbase-assembly-byo-hadoop/src/main/assembly/hadoop-three-compat.xml
@@ -0,0 +1,158 @@
+<?xml version="1.0"?>
+<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1
http://maven.apache.org/xsd/assembly-1.1.1.xsd";>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+ <!--This 'all' id is not appended to the produced bundle because we do this:
http://maven.apache.org/plugins/maven-assembly-plugin/faq.html#required-classifiers
-->
+ <id>bin</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <componentDescriptors>
+ <componentDescriptor>src/main/assembly/components.xml</componentDescriptor>
+ </componentDescriptors>
+ <!-- Include the generated LICENSE and NOTICE files -->
+ <files>
+ <file>
+
<source>${project.build.directory}/maven-shared-archive-resources/META-INF/LICENSE</source>
+ <outputDirectory>.</outputDirectory>
+ <destName>LICENSE.txt</destName>
+ <lineEnding>unix</lineEnding>
+ </file>
+ <file>
+ <source>${project.build.directory}/NOTICE.aggregate</source>
+ <outputDirectory>.</outputDirectory>
+ <destName>NOTICE.txt</destName>
+ <lineEnding>unix</lineEnding>
+ </file>
+ <file>
+ <source>${basedir}/src/main/resources/META-INF/LEGAL</source>
+ <outputDirectory>.</outputDirectory>
+ <destName>LEGAL</destName>
+ <lineEnding>unix</lineEnding>
+ </file>
+ </files>
+
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <!-- Exclude artifacts added in the sub-directories to avoid duplication
-->
+ <excludes>
+ <!-- Exclude the shaded jars that go in the lib/shaded-clients
directory -->
+ <exclude>org.apache.hbase:hbase-shaded-mapreduce</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-client-byo-hadoop</exclude>
+ <!-- Exclude the Ruby jar that goes in the lib/ruby directory -->
+ <exclude>org.jruby:jruby-complete</exclude>
+ <!-- Exclude jars that go into the lib/client-facing-thirdparty
directory -->
+ <exclude>com.github.stephenc.findbugs:findbugs-annotations</exclude>
+ <exclude>commons-logging:commons-logging</exclude>
+ <exclude>log4j:log4j</exclude>
+ <exclude>org.apache.htrace:htrace-core4</exclude>
+ <exclude>org.apache.htrace:htrace-core</exclude>
+ <exclude>org.apache.yetus:audience-annotations</exclude>
+ <exclude>org.slf4j:*</exclude>
+ <exclude>org.apache.logging.log4j:*</exclude>
+ <exclude>io.opentelemetry:*</exclude>
+ <!-- Exclude the opentelemetry agent that goes in the lib/trace
directory -->
+ <exclude>io.opentelemetry.javaagent:*</exclude>
+ <!-- Exclude jline2 which goes into lib/zk-client. jline3 is a
different artifact-->
+ <exclude>jline:jline</exclude>
+ <!-- These two exclusions are added here to preserve previous
behaviour, and are not included in sub-directories -->
+ <!-- TODO investigate using the provided mechanism for these -->
+ <exclude>com.sun.jersey:*</exclude>
+ <exclude>com.sun.jersey.contribs:*</exclude>
+ <!-- FIXME remove when jaxws-ri is removed from assembly -->
+ <exclude>com.sun.xml.ws:jaxws-ri:pom</exclude>
+ </excludes>
+ </dependencySet>
+ <!-- Add jruby-complete to hbase_home/lib/ruby.
+ Update JRUBY_PACKAGED_WITH_HBASE in bin/hbase and hbase.cmd if you
would like to update outputDirectory below -->
+ <dependencySet>
+ <outputDirectory>lib/ruby</outputDirectory>
+ <includes>
+ <include>org.jruby:jruby-complete</include>
+ </includes>
+ </dependencySet>
+ <!-- Include third party dependencies the shaded clients expose in the lib
directory
+ -->
+ <dependencySet>
+ <outputDirectory>lib/client-facing-thirdparty</outputDirectory>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <!-- Unfortunately, we have to whitelist these because Maven
+ currently doesn't use the dependency-reduced-pom after
+ the shaded module has done its thing. That means if we
+ did this as "transitives of the shaded modules" we'd
+ get a duplication of all the jars we already have in our
+ shaded artifacts. See MNG-5899.
+
+ Check that things match by listing files and making
+ sure the runtime scoped things are all present in the
+ tarball. e.g.:
+
+ for module in hbase-shaded-mapreduce hbase-shaded-client; do
+ mvn dependency:list -f
hbase-shaded/${module}/dependency-reduced-pom.xml
+ done | \
+ grep -E "runtime|compile" | \
+ grep -v -E "junit|(optional)" | \
+ cut -d ' ' -f 3- | \
+ sort -u
+
+ TODO we should check this in nightly
+
+ Alternatively, we could
+ stop waiting for Maven to fix this and build the client
+ tarball in a different build.
+ -->
+ <includes>
+ <!-- TODO Review and remove entries that are not present in any
supported HBase and Hadoop version -->
+ <include>com.github.stephenc.findbugs:findbugs-annotations</include>
+ <include>commons-logging:commons-logging</include>
+ <include>log4j:log4j</include>
+ <include>org.apache.htrace:htrace-core4</include>
+ <include>org.apache.htrace:htrace-core</include>
+ <include>org.apache.yetus:audience-annotations</include>
+ <include>org.slf4j:*</include>
+ <include>org.apache.logging.log4j:*</include>
+ <include>io.opentelemetry:*</include>
+ </includes>
+ </dependencySet>
+ <dependencySet>
+ <outputDirectory>lib/shaded-clients</outputDirectory>
+ <includes>
+ <include>org.apache.hbase:hbase-shaded-mapreduce</include>
+ <include>org.apache.hbase:hbase-shaded-client-byo-hadoop</include>
+ </includes>
+ </dependencySet>
+ <dependencySet>
+ <outputDirectory>lib/zkcli</outputDirectory>
+ <includes>
+ <!-- This is jline2 -->
+ <include>jline:jline</include>
+ </includes>
+ </dependencySet>
+ <dependencySet>
+ <outputDirectory>lib/trace</outputDirectory>
+ <includes>
+ <include>io.opentelemetry.javaagent:*</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
diff --git a/hbase-assembly-byo-hadoop/src/main/resources/META-INF/LEGAL
b/hbase-assembly-byo-hadoop/src/main/resources/META-INF/LEGAL
new file mode 100644
index 00000000000..9ac7f5a78b1
--- /dev/null
+++ b/hbase-assembly-byo-hadoop/src/main/resources/META-INF/LEGAL
@@ -0,0 +1,5 @@
+In Ruby's source distribution, this would describe a number of C source files
+that have different licenses than Ruby itself. None of those apply to JRuby,
+so we have this file here as a placeholder.
+
+For details of licensing of this collective work, see LICENSE
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
index c9db9a9ddee..4631d365172 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly/pom.xml
@@ -47,6 +47,7 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-mapreduce</artifactId>
</dependency>
+ <!-- Intra-project dependencies -->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-openssl</artifactId>
diff --git a/pom.xml b/pom.xml
index ac514d53938..3ceb1c3b071 100644
--- a/pom.xml
+++ b/pom.xml
@@ -738,6 +738,7 @@
<module>hbase-it</module>
<module>hbase-examples</module>
<module>hbase-assembly</module>
+ <module>hbase-assembly-byo-hadoop</module>
<module>hbase-testing-util</module>
<module>hbase-annotations</module>
<module>hbase-rest</module>
@@ -1027,6 +1028,11 @@
is what glassfish's jspC jar uses and that's where we get our own
need for a jsp-api.
-->
<!-- Intra-module dependencies -->
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-annotations</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-annotations</artifactId>
