This is an automated email from the ASF dual-hosted git repository. mmiklavcic pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push: new 895fe49 METRON-2274 Flatfile loader and summarizer mapreduce mode broken (mmiklavc) closes apache/metron#1525 895fe49 is described below commit 895fe493f38fd85abd3e0f14df507293b977ccfe Author: mmiklavc <michael.miklav...@gmail.com> AuthorDate: Fri Oct 4 13:15:30 2019 -0600 METRON-2274 Flatfile loader and summarizer mapreduce mode broken (mmiklavc) closes apache/metron#1525 --- .../metron-data-management/src/main/scripts/flatfile_loader.sh | 7 +++++-- .../metron-data-management/src/main/scripts/flatfile_summarizer.sh | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh index 5453bf4..8baa6c7 100755 --- a/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh +++ b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh @@ -36,8 +36,11 @@ export HBASE_CONF=${HBASE_CONF:-/etc/hbase/conf} export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS" if [ $(which hadoop) ] then - export HADOOP_CLASSPATH="$METRON_HOME/lib/$DM_JAR:$HBASE_CONF:$METRON_HOME/lib/$STELLAR_JAR" - hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME "$@" + # libjars needed to pass the stellar dep to the map and reduce jvm tasks + LIBJARS="$METRON_HOME/lib/$STELLAR_JAR,$LIBJARS" + # hadoop classpath used in the local jvm for the client application + export HADOOP_CLASSPATH="$HBASE_CONF:$METRON_HOME/lib/$STELLAR_JAR" + hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME -libjars ${LIBJARS} "$@" else echo "Warning: Metron cannot find the hadoop client on this node. This means that loading via Map Reduce will NOT function." CP=$METRON_HOME/lib/$DM_JAR:$HBASE_CONF:$METRON_HOME/lib/$STELLAR_JAR diff --git a/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh b/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh index 1279ecc..aa94ebb 100755 --- a/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh +++ b/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh @@ -43,6 +43,8 @@ then fi done export HADOOP_CLASSPATH + # need this bc Stellar is no longer in the uber jar deps for metron-data-management + LIBJARS="$METRON_HOME/lib/$STELLAR_JAR,$LIBJARS" hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME -libjars ${LIBJARS} "$@" else echo "Warning: Metron cannot find the hadoop client on this node. This means that loading via Map Reduce will NOT function."