This is an automated email from the ASF dual-hosted git repository.
mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/master by this push:
new 8cb9cbb Build shaded jar without Hadoop for M/R (#44)
8cb9cbb is described below
commit 8cb9cbbf08b173ed3e9ff188c6516def2eb1bc5b
Author: Mike Walch <mwa...@apache.org>
AuthorDate: Wed Nov 28 14:12:28 2018 -0500
Build shaded jar without Hadoop for M/R (#44)
* Hadoop jars get added to CLASSPATH when needed
---
Dockerfile | 46 ++++++++++++++++++++++++++--------------------
bin/cingest | 3 +--
bin/rwalk | 2 +-
conf/env.sh.example | 14 ++++++++++----
pom.xml | 4 ++--
5 files changed, 40 insertions(+), 29 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 85cce6e..4038764 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -18,24 +18,30 @@ FROM centos:7
RUN yum install -y java-1.8.0-openjdk-devel
ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk
-ARG TEST_JAR_VERSION
-ENV TEST_JAR_VERSION 2.0.0-SNAPSHOT
-ENV TEST_JAR_PATH /opt/accumulo-testing-${TEST_JAR_VERSION}-shaded.jar
-ENV ACCUMULO_CLIENT_PROPS /opt/conf/accumulo-client.properties
-ENV TEST_PROPS /opt/conf/accumulo-testing.properties
-ENV TEST_LOG4J /opt/conf/log4j.properties.example
-
-RUN mkdir /opt/bin
-RUN mkdir /opt/conf
-RUN touch /opt/conf/env.sh
-
-ADD ./conf/accumulo-client.properties /opt/conf/
-ADD ./conf/accumulo-testing.properties /opt/conf/
-ADD ./conf/log4j.properties.example /opt/conf/
-ADD ./bin/cingest /opt/bin
-ADD ./bin/rwalk /opt/bin
-ADD ./src/main/docker/docker-entry /opt/bin
-ADD ./target/accumulo-testing-${TEST_JAR_VERSION}-shaded.jar /opt/
-
-ENTRYPOINT ["/opt/bin/docker-entry"]
+ENV HADOOP_API_JAR /opt/at/hadoop-client-api.jar
+ENV HADOOP_RUNTIME_JAR /opt/at/hadoop-client-runtime.jar
+ENV TEST_JAR_PATH /opt/at/accumulo-testing-shaded.jar
+ENV ACCUMULO_CLIENT_PROPS /opt/at/conf/accumulo-client.properties
+ENV TEST_PROPS /opt/at/conf/accumulo-testing.properties
+ENV TEST_LOG4J /opt/at/conf/log4j.properties
+
+RUN mkdir /opt/at
+RUN mkdir /opt/at/bin
+RUN mkdir /opt/at/conf
+
+COPY ./conf/accumulo-client.properties /opt/at/conf/
+COPY ./conf/accumulo-testing.properties /opt/at/conf/
+COPY ./conf/log4j.properties* /opt/at/conf/
+RUN if [[ ! -f /opt/at/conf/log4j.properties ]]; then mv
/opt/at/conf/log4j.properties.example /opt/at/conf/log4j.properties; fi
+RUN touch /opt/at/conf/env.sh
+
+COPY ./bin/cingest /opt/at/bin
+COPY ./bin/rwalk /opt/at/bin
+COPY ./src/main/docker/docker-entry /opt/at/bin
+
+COPY ./target/accumulo-testing-shaded.jar /opt/at/
+COPY ./target/dependency/hadoop-client-api.jar /opt/at/
+COPY ./target/dependency/hadoop-client-runtime.jar /opt/at/
+
+ENTRYPOINT ["/opt/at/bin/docker-entry"]
CMD ["help"]
diff --git a/bin/cingest b/bin/cingest
index 5801a12..6cbbb40 100755
--- a/bin/cingest
+++ b/bin/cingest
@@ -77,12 +77,11 @@ case "$1" in
exit 1
esac
-export CLASSPATH="$TEST_JAR_PATH:$CLASSPATH"
+export
CLASSPATH="$TEST_JAR_PATH:$HADOOP_API_JAR:$HADOOP_RUNTIME_JAR:$CLASSPATH"
case "$1" in
verify|moru)
if [ ! -z $HADOOP_HOME ]; then
- export CLASSPATH=$HADOOP_HOME/etc/hadoop:$CLASSPATH
export HADOOP_USE_CLIENT_CLASSLOADER=true
"$HADOOP_HOME"/bin/yarn jar "$TEST_JAR_PATH" "$ci_main" "$TEST_PROPS"
"$ACCUMULO_CLIENT_PROPS"
else
diff --git a/bin/rwalk b/bin/rwalk
index 8a17b4e..30cf0a7 100755
--- a/bin/rwalk
+++ b/bin/rwalk
@@ -34,7 +34,7 @@ else
. "$at_home"/conf/env.sh.example
fi
-export CLASSPATH="$TEST_JAR_PATH:$CLASSPATH"
+export
CLASSPATH="$TEST_JAR_PATH:$HADOOP_API_JAR:$HADOOP_RUNTIME_JAR:$CLASSPATH"
randomwalk_main="org.apache.accumulo.testing.randomwalk.Framework"
diff --git a/conf/env.sh.example b/conf/env.sh.example
index a3368fc..0a2cdbe 100644
--- a/conf/env.sh.example
+++ b/conf/env.sh.example
@@ -42,21 +42,27 @@ fi
# Shaded test jar
# ===============
-# Version of shaded test jar
-export TEST_JAR_VERSION=2.0.0-SNAPSHOT
# Versions set below will be what is included in the shaded jar
export ACCUMULO_VERSION="`$ACCUMULO_HOME/bin/accumulo version`"
export HADOOP_VERSION="`hadoop version | head -n1 | awk '{print $2}'`"
export ZOOKEEPER_VERSION=3.4.9
# Path to shaded test jar
at_home=$( cd "$( dirname "$conf_dir" )" && pwd )
-export
TEST_JAR_PATH="${at_home}/target/accumulo-testing-${TEST_JAR_VERSION}-shaded.jar"
-if [ ! -f "$TEST_JAR_PATH" ]; then
+export TEST_JAR_PATH="${at_home}/target/accumulo-testing-shaded.jar"
+if [[ ! -f "$TEST_JAR_PATH" ]]; then
echo "Building $TEST_JAR_PATH"
cd "${at_home}" || exit 1
mvn clean package -P create-shade-jar -D skipTests -D
accumulo.version="$ACCUMULO_VERSION" -D hadoop.version="$HADOOP_VERSION" -D
zookeeper.version="$ZOOKEEPER_VERSION"
fi
+# Hadoop jars
+# ===========
+export HADOOP_API_JAR="${at_home}"/target/dependency/hadoop-client-api.jar
+export
HADOOP_RUNTIME_JAR="${at_home}"/target/dependency/hadoop-client-runtime.jar
+if [[ ! -f "$HADOOP_API_JAR" || ! -f "$HADOOP_RUNTIME_JAR" ]]; then
+ mvn dependency:copy-dependencies -Dmdep.stripVersion=true
-DincludeArtifactIds=hadoop-client-api,hadoop-client-runtime
-Dhadoop.version="$HADOOP_VERSION"
+fi
+
# Agitator
# ========
# Accumulo user
diff --git a/pom.xml b/pom.xml
index 8d4c940..a9c534f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -208,11 +208,13 @@
</goals>
<phase>package</phase>
<configuration>
+ <finalName>${project.artifactId}-shaded</finalName>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>shaded</shadedClassifierName>
<artifactSet>
<excludes>
<exclude>org.apache.accumulo:accumulo-native</exclude>
+ <exclude>org.apache.hadoop:*</exclude>
</excludes>
</artifactSet>
<filters>
@@ -222,8 +224,6 @@
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
- <!-- Twill 0.13 fails if shaded jar includes this
directory !-->
- <exclude>META-INF/license/*</exclude>
</excludes>
</filter>
</filters>
