This is an automated email from the ASF dual-hosted git repository.
mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-uno.git
The following commit(s) were added to refs/heads/master by this push:
new aba7629 Updates for Hadoop 3 (#200)
aba7629 is described below
commit aba762959fd5fb43b04f9d5b143e7a704622617f
Author: Mike Walch <[email protected]>
AuthorDate: Thu Oct 4 11:02:27 2018 -0400
Updates for Hadoop 3 (#200)
---
bin/impl/setup-hadoop.sh | 19 ++++++++++++++++---
conf/hadoop/{mapred-site.xml => mapred-site-2.xml} | 0
conf/hadoop/mapred-site.xml | 8 ++++++++
conf/hadoop/{slaves => workers} | 0
4 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/bin/impl/setup-hadoop.sh b/bin/impl/setup-hadoop.sh
index 8945ca9..5614fc2 100755
--- a/bin/impl/setup-hadoop.sh
+++ b/bin/impl/setup-hadoop.sh
@@ -36,23 +36,36 @@ print_to_console " * ResourceManager status:
http://localhost:8088/";
print_to_console " * view logs at $HADOOP_LOG_DIR"
rm -rf "$INSTALL"/hadoop-*
-rm -f "$HADOOP_LOG_DIR"/*
-rm -rf "$HADOOP_LOG_DIR"/application_*
+rm -rf "$HADOOP_LOG_DIR"/*
rm -rf "$DATA_DIR"/hadoop
mkdir -p "$HADOOP_LOG_DIR"
tar xzf "$DOWNLOADS/$HADOOP_TARBALL" -C "$INSTALL"
hadoop_conf="$HADOOP_HOME"/etc/hadoop
-cp "$UNO_HOME"/conf/hadoop/* "$hadoop_conf/"
+cp "$UNO_HOME"/conf/hadoop/core-site.xml "$hadoop_conf/"
+cp "$UNO_HOME"/conf/hadoop/hdfs-site.xml "$hadoop_conf/"
+cp "$UNO_HOME"/conf/hadoop/yarn-site.xml "$hadoop_conf/"
+cp "$UNO_HOME"/conf/hadoop/capacity-scheduler.xml "$hadoop_conf/"
+cp "$UNO_HOME"/conf/hadoop/masters "$hadoop_conf/"
+if [[ $HADOOP_VERSION =~ ^2\..*$ ]]; then
+ cp "$UNO_HOME"/conf/hadoop/workers "$hadoop_conf/slaves"
+ cp "$UNO_HOME"/conf/hadoop/mapred-site-2.xml "$hadoop_conf/mapred-site.xml"
+else
+ cp "$UNO_HOME"/conf/hadoop/workers "$hadoop_conf/"
+ cp "$UNO_HOME"/conf/hadoop/mapred-site.xml "$hadoop_conf/"
+fi
+
$SED "s#UNO_HOST#$UNO_HOST#g" "$hadoop_conf/core-site.xml"
"$hadoop_conf/hdfs-site.xml" "$hadoop_conf/yarn-site.xml"
$SED "s#DATA_DIR#$DATA_DIR#g" "$hadoop_conf/hdfs-site.xml"
"$hadoop_conf/yarn-site.xml" "$hadoop_conf/mapred-site.xml"
+$SED "s#HADOOP_HOME#$HADOOP_HOME#g" "$hadoop_conf/mapred-site.xml"
$SED "s#HADOOP_LOG_DIR#$HADOOP_LOG_DIR#g" "$hadoop_conf/yarn-site.xml"
$SED "s#YARN_NM_MEM_MB#$YARN_NM_MEM_MB#g" "$hadoop_conf/yarn-site.xml"
$SED "s#YARN_NM_CPU_VCORES#$YARN_NM_CPU_VCORES#g" "$hadoop_conf/yarn-site.xml"
echo "export JAVA_HOME=$JAVA_HOME" >> "$hadoop_conf/hadoop-env.sh"
echo "export HADOOP_LOG_DIR=$HADOOP_LOG_DIR" >> "$hadoop_conf/hadoop-env.sh"
+echo "export HADOOP_MAPRED_HOME=$HADOOP_HOME" >> "$hadoop_conf/hadoop-env.sh"
if [[ $HADOOP_VERSION =~ ^2\..*$ ]]; then
echo "export YARN_LOG_DIR=$HADOOP_LOG_DIR" >> "$hadoop_conf/yarn-env.sh"
fi
diff --git a/conf/hadoop/mapred-site.xml b/conf/hadoop/mapred-site-2.xml
similarity index 100%
copy from conf/hadoop/mapred-site.xml
copy to conf/hadoop/mapred-site-2.xml
diff --git a/conf/hadoop/mapred-site.xml b/conf/hadoop/mapred-site.xml
index f951767..c8d793f 100644
--- a/conf/hadoop/mapred-site.xml
+++ b/conf/hadoop/mapred-site.xml
@@ -29,4 +29,12 @@
<name>mapreduce.cluster.local.dir</name>
<value>file://DATA_DIR/hadoop/mapred/local</value>
</property>
+ <property>
+ <name>yarn.app.mapreduce.am.env</name>
+ <value>HADOOP_MAPRED_HOME=HADOOP_HOME</value>
+ </property>
+ <property>
+ <name>mapreduce.application.classpath</name>
+
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
+ </property>
</configuration>
diff --git a/conf/hadoop/slaves b/conf/hadoop/workers
similarity index 100%
rename from conf/hadoop/slaves
rename to conf/hadoop/workers
