Could anyone test Spark+Hadoop in yarn-client mode on a bare-metal and not
a docker instance? I am stuck in
NY with limited time/resources.
my ugly batch script to build a hadoop/spark sandbox:
#!/bin/bash
set -x
sudo apt-get update
if [ -f /etc/os-release ]; then
. /etc/os-release
else
echo "Unknown Linux Distribution."
exit 1
fi
case ${ID}-${VERSION_ID} in
ubuntu-14.04)
sudo apt-get install -y software-properties-common
sudo add-apt-repository -y ppa:openjdk-r/ppa
;;
*)
echo "supported OS ${ID}-${VERSION_ID}."
esac
sudo apt-get update
sudo apt-get install -y wget libsnappy1 libsnappy-dev openssl liblzo2-2
openjdk-8-jdk unzip netcat-openbsd apt-utils openssh-server
#mkdir -p ~/.ssh
#echo -e "Host *\n\tStrictHostKeyChecking no" > ~/.ssh/config
#ssh-keygen -q -N "" -t rsa -f ~/.ssh/id_rsa
#cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
sudo grep -q hadoop /etc/group
if [ "$?" -eq 0 ]; then
sudo addgroup hadoop
fi
sudo usermod -a -G hadoop $USER
#rm -rf source
mkdir source; cd $_
if [ "$1" != "pass" ] ; then
if [ $HOSTTYPE = "powerpc64le" ] ; then
wget
https://ci.bigtop.apache.org/view/ppc64le/job/Bigtop-trunk-packages-ppc64le/BUILD_ENVIRONMENTS=ubuntu-15.04-ppc64le,COMPONENTS=hadoop,label=ppc64le-slave/lastSuccessfulBuild/artifact/output/hadoop/*zip*/hadoop.zip
wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages-ppc64le/BUILD_ENVIRONMENTS=ubuntu-15.04-ppc64le,COMPONENTS=bigtop-groovy,label=ppc64le-slave/lastSuccessfulBuild/artifact/output/bigtop-groovy/bigtop-groovy_2.4.4-1_all.deb
wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages-ppc64le/BUILD_ENVIRONMENTS=ubuntu-15.04-ppc64le,COMPONENTS=bigtop-jsvc,label=ppc64le-slave/lastSuccessfulBuild/artifact/output/bigtop-jsvc/bigtop-jsvc_1.0.15-1_ppc64el.deb
wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages-ppc64le/BUILD_ENVIRONMENTS=ubuntu-15.04-ppc64le,COMPONENTS=bigtop-tomcat,label=ppc64le-slave/lastSuccessfulBuild/artifact/output/bigtop-tomcat/bigtop-tomcat_6.0.36-1_all.deb
wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages-ppc64le/BUILD_ENVIRONMENTS=ubuntu-15.04-ppc64le,COMPONENTS=bigtop-utils,label=ppc64le-slave/lastSuccessfulBuild/artifact/output/bigtop-utils/bigtop-utils_1.2.0-1_all.deb
wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages-ppc64le/BUILD_ENVIRONMENTS=ubuntu-15.04-ppc64le,COMPONENTS=zookeeper,label=ppc64le-slave/lastSuccessfulBuild/artifact/output/zookeeper/zookeeper_3.4.6-1_all.deb
#wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages/BUILD_ENVIRONMENTS=ubuntu-14.04,COMPONENTS=zookeeper,label=docker-slave/lastSuccessfulBuild/artifact/output/zookeeper/zookeeper_3.4.6-1_all.deb
wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages-ppc64le/BUILD_ENVIRONMENTS=ubuntu-15.04-ppc64le,COMPONENTS=spark,label=ppc64le-slave/lastSuccessfulBuild/artifact/*zip*/archive.zip
fi
if [ $HOSTTYPE = "x86_64" ] ; then
wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages/BUILD_ENVIRONMENTS=ubuntu-14.04,COMPONENTS=hadoop,label=docker-slave/lastSuccessfulBuild/artifact/output/hadoop/*zip*/hadoop.zip
wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages/BUILD_ENVIRONMENTS=ubuntu-14.04,COMPONENTS=bigtop-groovy,label=docker-slave/lastSuccessfulBuild/artifact/output/bigtop-groovy/bigtop-groovy_2.4.4-1_all.deb
wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages/BUILD_ENVIRONMENTS=ubuntu-14.04,COMPONENTS=bigtop-jsvc,label=docker-slave/lastSuccessfulBuild/artifact/output/bigtop-jsvc/bigtop-jsvc_1.0.15-1_amd64.deb
wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages/BUILD_ENVIRONMENTS=ubuntu-14.04,COMPONENTS=bigtop-tomcat,label=docker-slave/lastSuccessfulBuild/artifact/output/bigtop-tomcat/bigtop-tomcat_6.0.36-1_all.deb
wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages/BUILD_ENVIRONMENTS=ubuntu-14.04,COMPONENTS=bigtop-utils,label=docker-slave/lastSuccessfulBuild/artifact/output/bigtop-utils/bigtop-utils_1.2.0-1_all.deb
wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages/BUILD_ENVIRONMENTS=ubuntu-14.04,COMPONENTS=zookeeper,label=docker-slave/lastSuccessfulBuild/artifact/output/zookeeper/zookeeper_3.4.6-1_all.deb
#wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages/BUILD_ENVIRONMENTS=ubuntu-14.04,COMPONENTS=zookeeper,label=docker-slave/lastSuccessfulBuild/artifact/output/zookeeper/zookeeper_3.4.6-1_all.deb
wget
https://ci.bigtop.apache.org/job/Bigtop-trunk-packages/BUILD_ENVIRONMENTS=ubuntu-14.04,COMPONENTS=spark,label=docker-slave/lastSuccessfulBuild/artifact/*zip*/archive.zip
fi
fi
unzip hadoop.zip
unzip archive.zip
sudo dpkg -i *.deb
cd hadoop
sudo dpkg -i *.deb
cd ../archive/output/spark
sudo dpkg -i *.deb
cd ../../..
export HADOOP_PREFIX=/usr/lib/hadoop
export JAVA_HOME=`sudo find /usr/ -name java-8-openjdk-*`
export HADOOP_CONF_DIR=/etc/hadoop/conf
echo "export JAVA_HOME=`sudo find /usr/ -name java-8-openjdk-*`" | sudo tee
-a /etc/environment $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
$HADOOP_PREFIX/etc/hadoop/yarn-env.sh
echo "export HADOOP_CONF_DIR=/etc/hadoop/conf" | sudo tee -a
$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
$HADOOP_PREFIX/etc/hadoop/yarn-env.sh
echo "export HADOOP_PREFIX=/usr/lib/hadoop" | sudo tee -a
$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
$HADOOP_PREFIX/etc/hadoop/yarn-env.sh
echo "export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec" |
$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
$HADOOP_PREFIX/etc/hadoop/yarn-env.sh
echo "export HADOOP_LOGS=/usr/lib/hadoop/logs" | sudo tee -a
$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
$HADOOP_PREFIX/etc/hadoop/yarn-env.sh
echo "export HADOOP_COMMON_HOME=/usr/lib/hadoop" | sudo tee -a
$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
$HADOOP_PREFIX/etc/hadoop/yarn-env.sh
echo "export HADOOP_HDFS_HOME=/usr/lib/hadoop-hdfs" | sudo tee -a
$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
$HADOOP_PREFIX/etc/hadoop/yarn-env.sh
echo "export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce" | sudo tee -a
$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
$HADOOP_PREFIX/etc/hadoop/yarn-env.sh
echo "export HADOOP_YARN_HOME=/usr/lib/hadoop-yarn" | sudo tee -a
$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
$HADOOP_PREFIX/etc/hadoop/yarn-env.sh
sudo sed -i s/localhost/$HOSTNAME/ /etc/hadoop/conf/core-site.xml
sudo ps -aux | grep java | awk '{print $2}' | sudo xargs kill
sudo chown -R $USER:hadoop /usr/lib/hadoop*
sudo chown -R hdfs:hadoop /var/log/hadoop-hdfs*
sudo chown -R yarn:hadoop /var/log/hadoop-yarn*
sudo chown -R mapred:hadoop /var/log/hadoop-mapred*
sudo chown -R $USER:hadoop /etc/hadoop
sudo -u hdfs hdfs namenode -format -force
sudo rm -rf /var/lib/hadoop-hdfs/cache/hdfs/dfs/data
for x in `cd /etc/init.d ; ls hadoop-hdfs-*` ; do sudo service $x restart ;
done
sudo -u hdfs hadoop fs -mkdir -p
/tmp/hadoop-yarn/staging/history/done_intermediate
sudo -u hdfs hadoop fs -chown -R mapred:mapred /tmp/hadoop-yarn/staging
sudo -u hdfs hadoop fs -chmod -R 1777 /tmp
sudo -u hdfs hadoop fs -mkdir -p /var/log/hadoop-yarn
sudo -u hdfs hadoop fs -chown yarn:mapred /var/log/hadoop-yarn
sudo service hadoop-yarn-resourcemanager start
sudo service hadoop-yarn-nodemanager start
sudo service hadoop-mapreduce-historyserver start
sudo service hadoop-yarn-timelineserver restart
sudo -u hdfs hadoop fs -mkdir -p /user/$USER
sudo -u hdfs hadoop fs -chown $USER /user/$USER
cd ..
### Spark configuration
echo "export SPARK_MASTER_IP=`hostname`" |sudo tee -a
/etc/spark/conf/spark-env.sh
sudo chown -R $USER:hadoop /etc/spark
cp /etc/spark/conf/spark-defaults.conf.template
/etc/spark/conf/spark-defaults.conf
echo "spark.master spark://$(hostname):7077"
>>/etc/spark/conf/spark-defaults.conf
echo "spark.eventLog.enabled true"
>>/etc/spark/conf/spark-defaults.conf
echo "spark.eventLog.dir hdfs://$(hostname):8020/directory"
>>/etc/spark/conf/spark-defaults.conf
cp /etc/spark/conf/log4j.properties.template
/etc/spark/conf/log4j.properties
echo "log4j.rootCategory=ERROR, console">>/etc/spark/conf/log4j.properties
sudo -u hdfs hadoop fs -mkdir -p /directory
sudo -u hdfs hadoop fs -chown -R spark:hadoop /directory
sudo -u hdfs hdfs dfs -chmod -R 1777 /directory
sudo -u hdfs hdfs dfs -mkdir -p /var/log/spark/apps
sudo -u hdfs hdfs dfs -chown -R $USER:hadoop /var/log/spark
for x in `cd /etc/init.d ; ls spark-*` ; do sudo service $x restart ; done
#### BigTOP Services Status
GREEN='\033[0;32m'
NC='\033[0m' # No Color
printf ">>>> ${GREEN}Apache BigTop Spark${NC} Services Status\n"
for x in `cd /etc/init.d ; ls spark*` ; do sudo service $x status ; done
printf ">>>> ${GREEN}Apache BigTop Hadoop-HDFS${NC} Services Status\n"
for x in `cd /etc/init.d ; ls hadoop-hdfs*` ; do sudo service $x status ;
done
printf ">>>> ${GREEN}Apache BigTop Hadoop-MAPREDUCE${NC} Services Status\n"
for x in `cd /etc/init.d ; ls hadoop-map*` ; do sudo service $x status ;
done
printf ">>>> ${GREEN}Apache BigTop HADOOP-YARN${NC} Services Status\n"
sudo service hadoop-yarn-resourcemanager status
sudo service hadoop-yarn-nodemanager status
sudo service hadoop-mapreduce-historyserver status
sudo service hadoop-yarn-timelineserver status
if [ "$1" != "pass" ] ; then
rm -rf source
fi
On Thu, Feb 18, 2016 at 7:18 AM, MrAsanjar . <[email protected]> wrote:
> if anyone interested I have a script that builds the sandbox (tested in
> Docker).
>
> On Thu, Feb 18, 2016 at 2:01 AM, Konstantin Boudnik <[email protected]>
> wrote:
>
>> That'd be the perfect question to the EMR team, who did the work on this
>> new
>> version of the Spark. I fixed a few standalone issues with Spark 1.5.1,
>> but I
>> have no clue about YARN though...
>>
>> Cos
>>
>> On Wed, Feb 17, 2016 at 09:46PM, MrAsanjar . wrote:
>> > I have build a single node hadoop/spark sandbox based on the latest
>> > Apachue Bigtop 1.1.0 build. Spark in standalone mode + HDFS functions
>> > perfectly, however, fails if yarn-client/yarn-master mode is used as
>> follow:
>> >
>> > *>>spark-submit --class org.apache.spark.examples.SparkPi --master
>> > yarn-client /usr/lib/spark/lib/spark-examples-1.5.1-hadoop2.7.1.jar 10*
>> > 16/02/17 05:19:52 ERROR YarnClientSchedulerBackend: Yarn application has
>> > already exited with state FINISHED!
>> > Exception in thread "main" java.lang.IllegalStateException: Cannot call
>> > methods on a stopped SparkContext
>> > at org.apache.spark.SparkContext.org
>> > $apache$spark$SparkContext$$assertNotStopped(SparkContext.scala:104)
>> > ......
>> >
>> >
>> > Looking at yarn Application log file, there is a *RECEIVED SIGNAL 15:
>> > SIGTERM *termination signal from the the yarn container.
>> > >>*yarn logs -applicationId application_1455683261278_0001*
>> >
>> > YARN executor launch context:
>> > env:
>> > CLASSPATH ->
>> >
>> {{PWD}}<CPS>{{PWD}}/__spark__.jar<CPS>$HADOOP_CONF_DIR<CPS>$HADOOP_COMMON_HOME/*<CPS>$HADOOP_COMMON_HOME/lib/*<CPS>$HADOOP_HDFS_HOME/*<CPS>$HADOOP_HDFS_HOME/lib/*<CPS>$HADOOP_MAPRED_HOME/*<CPS>$HADOOP_MAPRED_HOME/lib/*<CPS>$HADOOP_YARN_HOME/*<CPS>$HADOOP_YARN_HOME/lib/*<CPS>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*<CPS>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*
>> > SPARK_LOG_URL_STDERR -> http://2657cd5dc2f6:8042/node/containerlogs
>> >
>> ===============================================================================
>> >
>> > 16/02/17 04:29:18 INFO impl.ContainerManagementProtocolProxy: Opening
>> proxy
>> > : 2657cd5dc2f6:33785
>> > 16/02/17 04:29:18 ERROR yarn.ApplicationMaster: *RECEIVED SIGNAL 15:
>> > SIGTERM*
>> > 16/02/17 04:29:18 INFO yarn.ApplicationMaster: Final app status:
>> UNDEFINED,
>> > exitCode: 0, (reason: Shutdown hook called before final status was
>> > reported.)
>> > 16/02/17 04:29:18 INFO yarn.ApplicationMaster: Unregistering
>> > ApplicationMaster with UNDEFINED (diag message: Shutdown hook called
>> before
>> > final status was reported.)
>> > 16/02/17 04:29:18 INFO impl.AMRMClientImpl: Waiting for application to
>> be
>> > successfully unregistered.
>> > 16/02/17 04:29:18 INFO yarn.ApplicationMaster: Deleting staging
>> directory
>> > .sparkStaging/application_1455683261278_0001
>> > 16/02/17 04:29:18 INFO util.ShutdownHookManager: Shutdown hook called
>> > End of LogType:stderr
>> >
>> >
>> > BTW, I have successfully tested hadoop yarn by running Teragen/Terasort
>> > mapreduce job.
>> > Before i start debugging, has anyone tested spark in yarn-client mode?
>>
>
>
