HBASE-20615 emphasize shaded artifacts in client tarball.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a7706ea0 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a7706ea0 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a7706ea0 Branch: refs/heads/HBASE-20331 Commit: a7706ea09659fb74702f961def130e682c317c68 Parents: 9eca900 Author: Sean Busbey <bus...@apache.org> Authored: Fri May 18 11:11:42 2018 -0500 Committer: Sean Busbey <bus...@apache.org> Committed: Mon Jun 11 15:51:02 2018 -0500 ---------------------------------------------------------------------- bin/hbase | 226 ++++++++++++++++--- bin/hbase-config.sh | 10 + hbase-assembly/pom.xml | 21 +- .../src/main/assembly/client-components.xml | 43 +++- hbase-assembly/src/main/assembly/client.xml | 131 ++++++----- hbase-assembly/src/main/assembly/components.xml | 3 +- .../src/main/assembly/hadoop-two-compat.xml | 80 ++++++- pom.xml | 6 + 8 files changed, 417 insertions(+), 103 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/a7706ea0/bin/hbase ---------------------------------------------------------------------- diff --git a/bin/hbase b/bin/hbase index 4f1c854..559a02e 100755 --- a/bin/hbase +++ b/bin/hbase @@ -71,11 +71,18 @@ if [ -d "${HBASE_HOME}/target" ]; then in_dev_env=true fi +# Detect if we are in the omnibus tarball +in_omnibus_tarball="false" +if [ -f "${HBASE_HOME}/bin/hbase-daemons.sh" ]; then + in_omnibus_tarball="true" +fi + read -d '' options_string << EOF Options: - --config DIR Configuration direction to use. Default: ./conf - --hosts HOSTS Override the list in 'regionservers' file - --auth-as-server Authenticate to ZooKeeper using servers configuration + --config DIR Configuration direction to use. Default: ./conf + --hosts HOSTS Override the list in 'regionservers' file + --auth-as-server Authenticate to ZooKeeper using servers configuration + --internal-classpath Skip attempting to use client facing jars (WARNING: unstable results between versions) EOF # if no args specified, show usage if [ $# = 0 ]; then @@ -87,16 +94,18 @@ if [ $# = 0 ]; then echo " shell Run the HBase shell" echo " hbck Run the hbase 'fsck' tool" echo " snapshot Tool for managing snapshots" - echo " wal Write-ahead-log analyzer" - echo " hfile Store file analyzer" - echo " zkcli Run the ZooKeeper shell" - echo " master Run an HBase HMaster node" - echo " regionserver Run an HBase HRegionServer node" - echo " zookeeper Run a ZooKeeper server" - echo " rest Run an HBase REST server" - echo " thrift Run the HBase Thrift server" - echo " thrift2 Run the HBase Thrift2 server" - echo " clean Run the HBase clean up script" + if [ "${in_omnibus_tarball}" = "true" ]; then + echo " wal Write-ahead-log analyzer" + echo " hfile Store file analyzer" + echo " zkcli Run the ZooKeeper shell" + echo " master Run an HBase HMaster node" + echo " regionserver Run an HBase HRegionServer node" + echo " zookeeper Run a ZooKeeper server" + echo " rest Run an HBase REST server" + echo " thrift Run the HBase Thrift server" + echo " thrift2 Run the HBase Thrift2 server" + echo " clean Run the HBase clean up script" + fi echo " classpath Dump hbase CLASSPATH" echo " mapredcp Dump CLASSPATH entries required by mapreduce" echo " pe Run PerformanceEvaluation" @@ -186,9 +195,99 @@ for f in $HBASE_HOME/hbase-jars/hbase*.jar; do fi done +#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH +# Allow this functionality to be disabled +if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then + HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) +fi + # Add libs to CLASSPATH -for f in $HBASE_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; +declare shaded_jar + +if [ "${INTERNAL_CLASSPATH}" != "true" ]; then + # find our shaded jars + declare shaded_client + declare shaded_client_byo_hadoop + declare shaded_mapreduce + for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-client*.jar; do + if [[ "${f}" =~ byo-hadoop ]]; then + shaded_client_byo_hadoop="${f}" + else + shaded_client="${f}" + fi + done + for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-mapreduce*.jar; do + shaded_mapreduce="${f}" + done + + # If command can use our shaded client, use it + declare -a commands_in_client_jar=("classpath" "version") + for c in "${commands_in_client_jar[@]}"; do + if [ "${COMMAND}" = "${c}" ]; then + if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_client_byo_hadoop}" + else + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_client}" + fi + break + fi + done + + # If command needs our shaded mapreduce, use it + # N.B "mapredcp" is not included here because in the shaded case it skips our built classpath + declare -a commands_in_mr_jar=("hbck" "snapshot" "canary" "regionsplitter" "pre-upgrade") + for c in "${commands_in_mr_jar[@]}"; do + if [ "${COMMAND}" = "${c}" ]; then + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_mapreduce}" + break + fi + done + + # Some commands specifically only can use shaded mapreduce when we'll get a full hadoop classpath at runtime + if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then + declare -a commands_in_mr_need_hadoop=("backup" "restore" "rowcounter" "cellcounter") + for c in "${commands_in_mr_need_hadoop[@]}"; do + if [ "${COMMAND}" = "${c}" ]; then + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_mapreduce}" + break + fi + done + fi +fi + + +if [ -n "${shaded_jar}" ] && [ -f "${shaded_jar}" ]; then + CLASSPATH="${CLASSPATH}:${shaded_jar}" +# fall through to grabbing all the lib jars and hope we're in the omnibus tarball +# +# N.B. shell specifically can't rely on the shaded artifacts because RSGroups is only +# available as non-shaded +# +# N.B. pe and ltt can't easily rely on shaded artifacts because they live in hbase-mapreduce:test-jar +# and need some other jars that haven't been relocated. Currently enumerating that list +# is too hard to be worth it. +# +else + for f in $HBASE_HOME/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; + done + # make it easier to check for shaded/not later on. + shaded_jar="" +fi +for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/*.jar; do + if [[ ! "${f}" =~ ^.*/htrace-core-3.*\.jar$ ]] && \ + [ "${f}" != "htrace-core.jar$" ] && \ + [[ ! "${f}" =~ ^.*/slf4j-log4j.*$ ]]; then + CLASSPATH="${CLASSPATH}:${f}" + fi done # default log directory & file @@ -201,9 +300,9 @@ fi function append_path() { if [ -z "$1" ]; then - echo $2 + echo "$2" else - echo $1:$2 + echo "$1:$2" fi } @@ -214,18 +313,34 @@ if [ "$HBASE_LIBRARY_PATH" != "" ]; then JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "$HBASE_LIBRARY_PATH") fi -#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH -# Allow this functionality to be disabled -if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then - HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) - if [ -f ${HADOOP_IN_PATH} ]; then - HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \ - org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null) - if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then - JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") - fi - CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`) +#If configured and available, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH +if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then + HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" "${HADOOP_IN_PATH}" \ + org.apache.hadoop.hbase.util.GetJavaProperty java.library.path) + if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then + JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") fi + CLASSPATH=$(append_path "${CLASSPATH}" "$(${HADOOP_IN_PATH} classpath 2>/dev/null)") +else + # Otherwise, if we're providing Hadoop we should include htrace 3 if we were built with a version that needs it. + for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/htrace-core-3*.jar "${HBASE_HOME}"/lib/client-facing-thirdparty/htrace-core.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi + done + # Some commands require special handling when using shaded jars. For these cases, we rely on hbase-shaded-mapreduce + # instead of hbase-shaded-client* because we make use of some IA.Private classes that aren't in the latter. However, + # we don't invoke them using the "hadoop jar" command so we need to ensure there are some Hadoop classes available + # when we're not doing runtime hadoop classpath lookup. + # + # luckily the set of classes we need are those packaged in the shaded-client. + for c in "${commands_in_mr_jar[@]}"; do + if [ "${COMMAND}" = "${c}" ] && [ -n "${shaded_jar}" ]; then + CLASSPATH="${CLASSPATH}:${shaded_client:?We couldn\'t find the shaded client jar even though we did find the shaded MR jar. for command ${COMMAND} we need both. please use --internal-classpath as a workaround.}" + break + fi + done fi # Add user-specified CLASSPATH last @@ -250,11 +365,11 @@ if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` fi if [ -d "$HBASE_HOME/build/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib) + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib") fi if [ -d "${HBASE_HOME}/lib/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/lib/native/${JAVA_PLATFORM}) + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "${HBASE_HOME}/lib/native/${JAVA_PLATFORM}") fi fi @@ -379,8 +494,30 @@ elif [ "$COMMAND" = "zkcli" ] ; then done elif [ "$COMMAND" = "backup" ] ; then CLASS='org.apache.hadoop.hbase.backup.BackupDriver' + if [ -n "${shaded_jar}" ] ; then + for f in "${HBASE_HOME}"/lib/hbase-backup*.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi + done + fi elif [ "$COMMAND" = "restore" ] ; then CLASS='org.apache.hadoop.hbase.backup.RestoreDriver' + if [ -n "${shaded_jar}" ] ; then + for f in "${HBASE_HOME}"/lib/hbase-backup*.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi + done + for f in "${HBASE_HOME}"/lib/commons-lang3*.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi + done + fi elif [ "$COMMAND" = "upgrade" ] ; then echo "This command was used to upgrade to HBase 0.96, it was removed in HBase 2.0.0." echo "Please follow the documentation at http://hbase.apache.org/book.html#upgrading."; @@ -451,9 +588,24 @@ elif [ "$COMMAND" = "clean" ] ; then "$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} $@ exit $? elif [ "$COMMAND" = "mapredcp" ] ; then + # If we didn't find a jar above, this will just be blank and the + # check below will then default back to the internal classpath. + shaded_jar="${shaded_mapreduce}" + if [ "${INTERNAL_CLASSPATH}" != "true" ] && [ -f "${shaded_jar}" ]; then + echo -n "${shaded_jar}" + for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/*.jar; do + if [[ ! "${f}" =~ ^.*/htrace-core-3.*\.jar$ ]] && \ + [ "${f}" != "htrace-core.jar$" ] && \ + [[ ! "${f}" =~ ^.*/slf4j-log4j.*$ ]]; then + echo -n ":${f}" + fi + done + echo "" + exit 0 + fi CLASS='org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool' elif [ "$COMMAND" = "classpath" ] ; then - echo $CLASSPATH + echo "$CLASSPATH" exit 0 elif [ "$COMMAND" = "pe" ] ; then CLASS='org.apache.hadoop.hbase.PerformanceEvaluation' @@ -500,8 +652,20 @@ else fi HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX" +# by now if we're running a command it means we need logging +for f in ${HBASE_HOME}/lib/client-facing-thirdparty/slf4j-log4j*.jar; do + if [ -f "${f}" ]; then + CLASSPATH="${CLASSPATH}:${f}" + break + fi +done + # Exec unless HBASE_NOEXEC is set. export CLASSPATH +if [ "${DEBUG}" = "true" ]; then + echo "classpath=${CLASSPATH}" >&2 + HBASE_OPTS="${HBASE_OPTS} -Xdiag" +fi if [ "${HBASE_NOEXEC}" != "" ]; then "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@" http://git-wip-us.apache.org/repos/asf/hbase/blob/a7706ea0/bin/hbase-config.sh ---------------------------------------------------------------------- diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh index 2e95ae7..1054751 100644 --- a/bin/hbase-config.sh +++ b/bin/hbase-config.sh @@ -84,6 +84,16 @@ do exit 1 fi shift + elif [ "--internal-classpath" = "$1" ] + then + shift + # shellcheck disable=SC2034 + INTERNAL_CLASSPATH="true" + elif [ "--debug" = "$1" ] + then + shift + # shellcheck disable=SC2034 + DEBUG="true" else # Presume we are at end of options and break break http://git-wip-us.apache.org/repos/asf/hbase/blob/a7706ea0/hbase-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml index 4fa859a..fd1415f 100644 --- a/hbase-assembly/pom.xml +++ b/hbase-assembly/pom.xml @@ -190,6 +190,19 @@ </plugins> </build> <dependencies> + <!-- client artifacts for downstream use --> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-shaded-client</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-shaded-client-byo-hadoop</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-shaded-mapreduce</artifactId> + </dependency> <!-- Intra-project dependencies --> <dependency> <groupId>org.apache.hbase</groupId> @@ -316,14 +329,6 @@ <groupId>jline</groupId> <artifactId>jline</artifactId> </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-shaded-client</artifactId> - </dependency> - <dependency> - <groupId>org.apache.hbase</groupId> - <artifactId>hbase-shaded-mapreduce</artifactId> - </dependency> </dependencies> <profiles> <profile> http://git-wip-us.apache.org/repos/asf/hbase/blob/a7706ea0/hbase-assembly/src/main/assembly/client-components.xml ---------------------------------------------------------------------- diff --git a/hbase-assembly/src/main/assembly/client-components.xml b/hbase-assembly/src/main/assembly/client-components.xml index 2fd1b57..2369f28 100644 --- a/hbase-assembly/src/main/assembly/client-components.xml +++ b/hbase-assembly/src/main/assembly/client-components.xml @@ -32,7 +32,8 @@ <directory>${project.basedir}/..</directory> <outputDirectory>.</outputDirectory> <includes> - <include>CHANGES.txt</include> + <include>CHANGES.md</include> + <include>RELEASENOTES.md</include> <include>README.txt</include> </includes> <fileMode>0644</fileMode> @@ -56,7 +57,6 @@ <include>hbase-config.sh</include> <include>hbase-jruby</include> <include>hirb.rb</include> - <include></include> </includes> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> @@ -88,5 +88,44 @@ <include>*.dylib</include> </includes> </fileSet> + <!-- This is only necessary until maven fixes the intra-project dependency bug + in maven 3.0. Until then, we have to include the test jars for sub-projects. When + fixed, the below dependencySet stuff is sufficient for pulling in the test jars as + well, as long as they are added as dependencies in this project. Right now, we only + have 1 submodule to accumulate, but we can copy/paste as necessary until maven is + fixed. --> + <!-- Used by PE and ltt --> + <fileSet> + <directory>${project.basedir}/../hbase-server/target/</directory> + <outputDirectory>lib</outputDirectory> + <includes> + <include>${server.test.jar}</include> + </includes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../hbase-mapreduce/target/</directory> + <outputDirectory>lib</outputDirectory> + <includes> + <include>${mapreduce.test.jar}</include> + </includes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../hbase-common/target/</directory> + <outputDirectory>lib</outputDirectory> + <includes> + <include>${common.test.jar}</include> + </includes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../hbase-zookeeper/target/</directory> + <outputDirectory>lib</outputDirectory> + <includes> + <include>${zookeeper.test.jar}</include> + </includes> + <fileMode>0644</fileMode> + </fileSet> </fileSets> </component> http://git-wip-us.apache.org/repos/asf/hbase/blob/a7706ea0/hbase-assembly/src/main/assembly/client.xml ---------------------------------------------------------------------- diff --git a/hbase-assembly/src/main/assembly/client.xml b/hbase-assembly/src/main/assembly/client.xml index 7951961..c875b95 100644 --- a/hbase-assembly/src/main/assembly/client.xml +++ b/hbase-assembly/src/main/assembly/client.xml @@ -31,73 +31,37 @@ <componentDescriptor>src/main/assembly/client-components.xml</componentDescriptor> </componentDescriptors> <moduleSets> + <!-- include regular jars so the shell can use them --> <moduleSet> <useAllReactorProjects>true</useAllReactorProjects> <includes> - <!-- Keep this list sorted by name --> - <include>org.apache.hbase:hbase-annotations</include> - <include>org.apache.hbase:hbase-client</include> - <include>org.apache.hbase:hbase-common</include> - <include>org.apache.hbase:hbase-hadoop-compat</include> - <include>org.apache.hbase:hbase-hadoop2-compat</include> - <include>org.apache.hbase:hbase-mapreduce</include> - <include>org.apache.hbase:hbase-metrics</include> - <include>org.apache.hbase:hbase-metrics-api</include> - <include>org.apache.hbase:hbase-procedure</include> - <include>org.apache.hbase:hbase-protocol</include> - <include>org.apache.hbase:hbase-protocol-shaded</include> - <include>org.apache.hbase:hbase-server</include> - <include>org.apache.hbase:hbase-zookeeper</include> + <include>org.apache.hbase:hbase-shell</include> </includes> - <!-- Binaries for the dependencies also go in the hbase-jars directory --> <binaries> - <outputDirectory>lib</outputDirectory> <unpack>false</unpack> - <dependencySets> - <!-- Exclude jruby-complete from hbase_home/lib --> - <dependencySet> + <outputDirectory>lib</outputDirectory> + <dependencySets> + <dependencySet> + <!-- Exclude libraries that we put in their own dirs under lib/ --> <excludes> + <exclude>org.jruby:jruby-complete</exclude> <exclude>com.sun.jersey:*</exclude> <exclude>com.sun.jersey.contribs:*</exclude> - <!-- Exclude jars which typical clients don't need --> - <exclude>org.apache.hbase:hbase-external-blockcache</exclude> - <exclude>org.apache.hbase:hbase-http</exclude> - <exclude>org.apache.hbase:hbase-replication</exclude> - <exclude>org.apache.hbase:hbase-rest</exclude> - <exclude>org.apache.hbase:hbase-rsgroup</exclude> - <exclude>org.apache.hbase:hbase-shaded-client</exclude> - <exclude>org.apache.hbase:hbase-shaded-mapreduce</exclude> - <!-- At present, hbase-shell doesn't actually contain - any Java code we need to include. Ruby files are - copied elsewhere in this descriptor. --> - <exclude>org.apache.hbase:hbase-shell</exclude> - <exclude>org.apache.hbase:hbase-thrift</exclude> - <exclude>org.jruby:jruby-complete</exclude> + <exclude>jline:jline</exclude> + <exclude>com.github.stephenc.findbugs:findbugs-annotations</exclude> + <exclude>commons-logging:commons-logging</exclude> + <exclude>log4j:log4j</exclude> + <exclude>org.apache.hbase:hbase-shaded-client</exclude> + <exclude>org.apache.hbase:hbase-shaded-client-byo-hadoop</exclude> + <exclude>org.apache.hbase:hbase-shaded-mapreduce</exclude> + <exclude>org.apache.htrace:htrace-core4</exclude> + <exclude>org.apache.htrace:htrace-core</exclude> + <exclude>org.apache.yetus:audience-annotations</exclude> + <exclude>org.slf4j:slf4j-api</exclude> + <exclude>org.slf4j:slf4j-log4j12</exclude> </excludes> - </dependencySet> - </dependencySets> - </binaries> - </moduleSet> - <moduleSet> - <useAllReactorProjects>true</useAllReactorProjects> - <includes> - <include>org.apache.hbase:hbase-shaded-client</include> - <include>org.apache.hbase:hbase-shaded-mapreduce</include> - </includes> - <!-- Binaries for the dependencies also go in the hbase-jars directory --> - <binaries> - <outputDirectory>shaded-lib</outputDirectory> - <unpack>false</unpack> - <dependencySets> - <dependencySet> - <includes> - <!-- Keep this list sorted by name --> - <include>org.apache.hbase:hbase-shaded-client</include> - <include>org.apache.hbase:hbase-shaded-mapreduce</include> - </includes> - <useTransitiveDependencies>false</useTransitiveDependencies> - </dependencySet> - </dependencySets> + </dependencySet> + </dependencySets> </binaries> </moduleSet> </moduleSets> @@ -123,15 +87,66 @@ </file> </files> + <dependencySets> + <dependencySet> + <outputDirectory>lib/shaded-clients</outputDirectory> + <includes> + <include>org.apache.hbase:hbase-shaded-client</include> + <include>org.apache.hbase:hbase-shaded-mapreduce</include> + <include>org.apache.hbase:hbase-shaded-client-byo-hadoop</include> + </includes> + </dependencySet> <!-- Add jruby-complete to hbase_home/lib/ruby. Update JRUBY_PACKAGED_WITH_HBASE in bin/hbase and hbase.cmd if you would like to update outputDirectory below --> - <dependencySets> <dependencySet> <outputDirectory>lib/ruby</outputDirectory> <includes> <include>org.jruby:jruby-complete</include> </includes> </dependencySet> + <!-- Include third party dependencies the shaded clients expose in the lib directory + N.B. this will conflict with the omnibus tarball but these should be precisely + the same artifacts so blind overwrite of either should be fine. + --> + <dependencySet> + <outputDirectory>lib/client-facing-thirdparty</outputDirectory> + <useTransitiveDependencies>true</useTransitiveDependencies> + <!-- Unfortunately, we have to whitelist these because Maven + currently doesn't use the dependency-reduced-pom after + the shaded module has done its thing. That means if we + did this as "transitives of the shaded modules" we'd + get a duplication of all the jars we already have in our + shaded artifacts. See MNG-5899. + + Check that things match by listing files and making + sure the runtime scoped things are all present in the + tarball. e.g.: + + for module in hbase-shaded-mapreduce hbase-shaded-client; do + mvn dependency:list -f hbase-shaded/${module}/dependency-reduced-pom.xml + done | \ + grep -E "runtime|compile" | \ + grep -v -E "junit|(optional)" | \ + cut -d ' ' -f 3- | \ + sort -u + + TODO we should check this in nightly + + Alternatively, we could + stop waiting for Maven to fix this and build the client + tarball in a different build. + --> + <includes> + <include>com.github.stephenc.findbugs:findbugs-annotations</include> + <include>commons-logging:commons-logging</include> + <include>log4j:log4j</include> + <include>org.apache.htrace:htrace-core4</include> + <include>org.apache.htrace:htrace-core</include> + <include>org.apache.yetus:audience-annotations</include> + <include>org.slf4j:slf4j-api</include> + <include>org.slf4j:slf4j-log4j12</include> + </includes> + </dependencySet> </dependencySets> </assembly> http://git-wip-us.apache.org/repos/asf/hbase/blob/a7706ea0/hbase-assembly/src/main/assembly/components.xml ---------------------------------------------------------------------- diff --git a/hbase-assembly/src/main/assembly/components.xml b/hbase-assembly/src/main/assembly/components.xml index 59502d6..2eb16e7 100644 --- a/hbase-assembly/src/main/assembly/components.xml +++ b/hbase-assembly/src/main/assembly/components.xml @@ -37,7 +37,8 @@ <directory>${project.basedir}/..</directory> <outputDirectory>.</outputDirectory> <includes> - <include>CHANGES.txt</include> + <include>CHANGES.md</include> + <include>RELEASENOTES.md</include> <include>README.txt</include> </includes> <fileMode>0644</fileMode> http://git-wip-us.apache.org/repos/asf/hbase/blob/a7706ea0/hbase-assembly/src/main/assembly/hadoop-two-compat.xml ---------------------------------------------------------------------- diff --git a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml index 69a800b..7243783 100644 --- a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml +++ b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml @@ -66,18 +66,51 @@ <outputDirectory>lib</outputDirectory> <unpack>false</unpack> <dependencySets> - <!-- Exclude jruby-complete from hbase_home/lib --> <dependencySet> + <!-- Exclude libraries that we put in their own dirs under lib/ --> <excludes> <exclude>org.jruby:jruby-complete</exclude> <exclude>com.sun.jersey:*</exclude> <exclude>com.sun.jersey.contribs:*</exclude> <exclude>jline:jline</exclude> + <exclude>org.apache.hbase:hbase-shaded-client</exclude> + <exclude>org.apache.hbase:hbase-shaded-client-byo-hadoop</exclude> + <exclude>org.apache.hbase:hbase-shaded-mapreduce</exclude> + <exclude>com.github.stephenc.findbugs:findbugs-annotations</exclude> + <exclude>commons-logging:commons-logging</exclude> + <exclude>log4j:log4j</exclude> + <exclude>org.apache.htrace:htrace-core4</exclude> + <exclude>org.apache.htrace:htrace-core</exclude> + <exclude>org.apache.yetus:audience-annotations</exclude> + <exclude>org.slf4j:slf4j-api</exclude> + <exclude>org.slf4j:slf4j-log4j12</exclude> </excludes> </dependencySet> </dependencySets> </binaries> </moduleSet> + <!-- Include shaded clients in their own directory --> + <moduleSet> + <useAllReactorProjects>true</useAllReactorProjects> + <includes> + <include>org.apache.hbase:hbase-shaded-client</include> + <include>org.apache.hbase:hbase-shaded-mapreduce</include> + <include>org.apache.hbase:hbase-shaded-client-byo-hadoop</include> + </includes> + <binaries> + <outputDirectory>lib/shaded-clients</outputDirectory> + <unpack>false</unpack> + <dependencySets> + <dependencySet> + <includes> + <include>org.apache.hbase:hbase-shaded-client</include> + <include>org.apache.hbase:hbase-shaded-mapreduce</include> + <include>org.apache.hbase:hbase-shaded-client-byo-hadoop</include> + </includes> + </dependencySet> + </dependencySets> + </binaries> + </moduleSet> </moduleSets> <!-- Include the generated LICENSE and NOTICE files --> <files> @@ -101,15 +134,56 @@ </file> </files> - <!-- Add jruby-complete to hbase_home/lib/ruby. - Update JRUBY_PACKAGED_WITH_HBASE in bin/hbase and hbase.cmd if you would like to update outputDirectory below --> <dependencySets> + <!-- Add jruby-complete to hbase_home/lib/ruby. + Update JRUBY_PACKAGED_WITH_HBASE in bin/hbase and hbase.cmd if you would like to update outputDirectory below --> <dependencySet> <outputDirectory>lib/ruby</outputDirectory> <includes> <include>org.jruby:jruby-complete</include> </includes> </dependencySet> + <!-- Include third party dependencies the shaded clients expose in the lib directory + --> + <dependencySet> + <outputDirectory>lib/client-facing-thirdparty</outputDirectory> + <useTransitiveDependencies>true</useTransitiveDependencies> + <!-- Unfortunately, we have to whitelist these because Maven + currently doesn't use the dependency-reduced-pom after + the shaded module has done its thing. That means if we + did this as "transitives of the shaded modules" we'd + get a duplication of all the jars we already have in our + shaded artifacts. See MNG-5899. + + Check that things match by listing files and making + sure the runtime scoped things are all present in the + tarball. e.g.: + + for module in hbase-shaded-mapreduce hbase-shaded-client; do + mvn dependency:list -f hbase-shaded/${module}/dependency-reduced-pom.xml + done | \ + grep -E "runtime|compile" | \ + grep -v -E "junit|(optional)" | \ + cut -d ' ' -f 3- | \ + sort -u + + TODO we should check this in nightly + + Alternatively, we could + stop waiting for Maven to fix this and build the client + tarball in a different build. + --> + <includes> + <include>com.github.stephenc.findbugs:findbugs-annotations</include> + <include>commons-logging:commons-logging</include> + <include>log4j:log4j</include> + <include>org.apache.htrace:htrace-core4</include> + <include>org.apache.htrace:htrace-core</include> + <include>org.apache.yetus:audience-annotations</include> + <include>org.slf4j:slf4j-api</include> + <include>org.slf4j:slf4j-log4j12</include> + </includes> + </dependencySet> <dependencySet> <outputDirectory>lib/zkcli</outputDirectory> <includes> http://git-wip-us.apache.org/repos/asf/hbase/blob/a7706ea0/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index b9a1b77..ea02fe8 100755 --- a/pom.xml +++ b/pom.xml @@ -1555,6 +1555,7 @@ <annotations.test.jar>hbase-annotations-${project.version}-tests.jar</annotations.test.jar> <rsgroup.test.jar>hbase-rsgroup-${project.version}-tests.jar</rsgroup.test.jar> <mapreduce.test.jar>hbase-mapreduce-${project.version}-tests.jar</mapreduce.test.jar> + <zookeeper.test.jar>hbase-zookeeper-${project.version}-tests.jar</zookeeper.test.jar> <shell-executable>bash</shell-executable> <surefire.version>2.21.0</surefire.version> <surefire.provider>surefire-junit47</surefire.provider> @@ -1860,6 +1861,11 @@ </dependency> <dependency> <groupId>org.apache.hbase</groupId> + <artifactId>hbase-shaded-client-byo-hadoop</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> <artifactId>hbase-shaded-mapreduce</artifactId> <version>${project.version}</version> </dependency>
