This is an automated email from the ASF dual-hosted git repository.
mmiklavcic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 895fe49 METRON-2274 Flatfile loader and summarizer mapreduce mode
broken (mmiklavc) closes apache/metron#1525
895fe49 is described below
commit 895fe493f38fd85abd3e0f14df507293b977ccfe
Author: mmiklavc <[email protected]>
AuthorDate: Fri Oct 4 13:15:30 2019 -0600
METRON-2274 Flatfile loader and summarizer mapreduce mode broken (mmiklavc)
closes apache/metron#1525
---
.../metron-data-management/src/main/scripts/flatfile_loader.sh | 7 +++++--
.../metron-data-management/src/main/scripts/flatfile_summarizer.sh | 2 ++
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git
a/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
index 5453bf4..8baa6c7 100755
--- a/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
+++ b/metron-platform/metron-data-management/src/main/scripts/flatfile_loader.sh
@@ -36,8 +36,11 @@ export HBASE_CONF=${HBASE_CONF:-/etc/hbase/conf}
export HADOOP_OPTS="$HADOOP_OPTS $METRON_JVMFLAGS"
if [ $(which hadoop) ]
then
- export
HADOOP_CLASSPATH="$METRON_HOME/lib/$DM_JAR:$HBASE_CONF:$METRON_HOME/lib/$STELLAR_JAR"
- hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME "$@"
+ # libjars needed to pass the stellar dep to the map and reduce jvm tasks
+ LIBJARS="$METRON_HOME/lib/$STELLAR_JAR,$LIBJARS"
+ # hadoop classpath used in the local jvm for the client application
+ export HADOOP_CLASSPATH="$HBASE_CONF:$METRON_HOME/lib/$STELLAR_JAR"
+ hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME -libjars ${LIBJARS} "$@"
else
echo "Warning: Metron cannot find the hadoop client on this node. This
means that loading via Map Reduce will NOT function."
CP=$METRON_HOME/lib/$DM_JAR:$HBASE_CONF:$METRON_HOME/lib/$STELLAR_JAR
diff --git
a/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh
b/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh
index 1279ecc..aa94ebb 100755
---
a/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh
+++
b/metron-platform/metron-data-management/src/main/scripts/flatfile_summarizer.sh
@@ -43,6 +43,8 @@ then
fi
done
export HADOOP_CLASSPATH
+ # need this bc Stellar is no longer in the uber jar deps for
metron-data-management
+ LIBJARS="$METRON_HOME/lib/$STELLAR_JAR,$LIBJARS"
hadoop jar $METRON_HOME/lib/$DM_JAR $CLASSNAME -libjars ${LIBJARS} "$@"
else
echo "Warning: Metron cannot find the hadoop client on this node. This
means that loading via Map Reduce will NOT function."
