Repository: incubator-myriad Updated Branches: refs/heads/master 9f0fa15bf -> fed956b86
Updated Vagrantfile to work with the supported Mesos and Hadoop versi⦠â¦ons. Broke provisioning into separate scripts in the ./vagrant directory. Updated vagrant documentation Added .npmrc that eliminates node symbolic links so build will work under Vagrant. This closes: #43 Review: https://github.com/apache/incubator-myriad/pull/43 Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/fed956b8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/fed956b8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/fed956b8 Branch: refs/heads/master Commit: fed956b869bccb5b807221b5ed41e08d57926fe6 Parents: 9f0fa15 Author: klucar <klu...@gmail.com> Authored: Wed Nov 11 16:58:44 2015 -0800 Committer: smarella <smare...@maprtech.com> Committed: Wed Nov 11 16:58:44 2015 -0800 ---------------------------------------------------------------------- Vagrantfile | 123 ++++++++++--------- docs/vagrant.md | 51 ++------ .../src/main/resources/webapp/.npmrc | 1 + setup-yarn-1.sh | 18 --- setup-yarn-2.sh | 64 ---------- shutdown.sh | 10 -- vagrant/format_namenode.sh | 23 ++++ vagrant/install_default_jdk.sh | 32 +++++ vagrant/install_docker.sh | 25 ++++ vagrant/install_hadoop.sh | 115 +++++++++++++++++ vagrant/install_mesos.sh | 43 +++++++ vagrant/shutdown.sh | 30 +++++ vagrant/start_datanode.sh | 24 ++++ vagrant/start_historyserver.sh | 23 ++++ vagrant/start_mesos_master.sh | 34 +++++ vagrant/start_mesos_slave.sh | 38 ++++++ vagrant/start_namenode.sh | 27 ++++ 17 files changed, 488 insertions(+), 193 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/Vagrantfile ---------------------------------------------------------------------- diff --git a/Vagrantfile b/Vagrantfile index 88039f8..7102fb9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,80 +1,81 @@ # -*- mode: ruby -*- # # vi: set ft=ruby : -VAGRANTFILE_API_VERSION = "2" - -$provision_script = <<SCRIPT - -PREFIX="PROVISIONER:" - -set -e - -echo "${PREFIX} Installing pre-reqs..." - -# For installing Java 8 -add-apt-repository ppa:webupd8team/java - -# For Mesos -apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF -DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') -CODENAME=$(lsb_release -cs) -echo "deb http://repos.mesosphere.io/${DISTRO} ${CODENAME} main" | sudo tee /etc/apt/sources.list.d/mesosphere.list +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# -apt-get -y update -echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections -apt-get -y install oracle-java8-installer -apt-get -y install oracle-java8-set-default -apt-get -y install libcurl3 -apt-get -y install zookeeperd -apt-get -y install aria2 +VAGRANTFILE_API_VERSION = "2" MESOS_VERSION="0.21.1" -echo "${PREFIX}Installing mesos version: ${MESOS_VERSION}..." -apt-get -y install mesos -echo "Done" - -ln -s /usr/lib/jvm/java-8-oracle/jre/lib/amd64/server/libjvm.so /usr/lib/libjvm.so - -echo "${PREFIX}Starting mesos master" -start mesos-master - -echo "${PREFIX}Starting mesos slave" -start mesos-slave - -echo "${PREFIX}Successfully provisioned machine for Myriad development" - -SCRIPT +HADOOP_VERSION="2.7.0" +#PRIVATE_IP="10.141.141.20" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.box = "ubuntu/trusty64" config.vm.box_url = "https://vagrantcloud.com/ubuntu/boxes/trusty64" + #config.vm.hostname = "master" + #config.vm.network :private_network, ip: "#{PRIVATE_IP}" - # Consistent IP for demos and tutorials (10.141.141.10 is mesosphere playa address) - config.vm.network :private_network, ip: "10.141.141.20" + # Configure VM resources + config.vm.provider :virtualbox do |vb| + vb.name = "myriad-dev" + vb.customize ["modifyvm", :id, "--memory", "4096"] + vb.customize ["modifyvm", :id, "--cpus", "2"] + #vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + #vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"] + end + + if Vagrant.has_plugin?("vagrant-cachier") + # Configure cached packages to be shared between instances of the same base box. + config.cache.scope = :box + end # Forward mesos master and slave ports - config.vm.network "forwarded_port", guest: 5005, host: 5005 - config.vm.network "forwarded_port", guest: 5050, host: 5050 - config.vm.network "forwarded_port", guest: 5051, host: 5051 + #config.vm.network "forwarded_port", guest: 5005, host: 5005 + #config.vm.network "forwarded_port", guest: 5050, host: 5050 + #config.vm.network "forwarded_port", guest: 5051, host: 5051 # Forward myriad web and admin ports - config.vm.network "forwarded_port", guest: 8080, host: 8080 - config.vm.network "forwarded_port", guest: 8081, host: 8081 + #config.vm.network "forwarded_port", guest: 8080, host: 8080 + #config.vm.network "forwarded_port", guest: 8081, host: 8081 # Forward YARN/Hadoop ports - config.vm.network "forwarded_port", guest: 50070, host: 50070 - config.vm.network "forwarded_port", guest: 50075, host: 50075 - config.vm.network "forwarded_port", guest: 8088, host: 8088 - config.vm.network "forwarded_port", guest: 8042, host: 8042 - config.vm.network "forwarded_port", guest: 19888, host: 19888 - config.vm.network "forwarded_port", guest: 8192, host: 8192 - config.vm.network "forwarded_port", guest: 2181, host: 2181 + #config.vm.network "forwarded_port", guest: 50070, host: 50070 + #config.vm.network "forwarded_port", guest: 50075, host: 50075 + #config.vm.network "forwarded_port", guest: 8088, host: 8088 + #config.vm.network "forwarded_port", guest: 8042, host: 8042 + #config.vm.network "forwarded_port", guest: 19888, host: 19888 + #config.vm.network "forwarded_port", guest: 8192, host: 8192 + #config.vm.network "forwarded_port", guest: 2181, host: 2181 + + # install software + config.vm.provision "shell", path: "vagrant/install_default_jdk.sh" + config.vm.provision "shell", path: "vagrant/install_mesos.sh", args: ["#{MESOS_VERSION}"] + config.vm.provision "shell", path: "vagrant/install_docker.sh" + config.vm.provision "shell", path: "vagrant/install_hadoop.sh", args: ["#{HADOOP_VERSION}"] + + # start things up + config.vm.provision "shell", path: "vagrant/start_mesos_master.sh" + config.vm.provision "shell", path: "vagrant/start_mesos_slave.sh" + config.vm.provision "shell", path: "vagrant/format_namenode.sh" + config.vm.provision "shell", path: "vagrant/start_namenode.sh" + config.vm.provision "shell", path: "vagrant/start_datanode.sh" + config.vm.provision "shell", path: "vagrant/start_historyserver.sh" - config.vm.provision "shell", inline: $provision_script - - # Configure VM resources - config.vm.provider :virtualbox do |vb| - vb.customize ["modifyvm", :id, "--memory", "4096"] - vb.customize ["modifyvm", :id, "--cpus", "2"] - end end http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/docs/vagrant.md ---------------------------------------------------------------------- diff --git a/docs/vagrant.md b/docs/vagrant.md index 2762ef9..0c71ea7 100644 --- a/docs/vagrant.md +++ b/docs/vagrant.md @@ -15,7 +15,14 @@ To start the cluster run following: vagrant up ``` -At this point the VM will have a single node mesos cluster running. +At this point the VM will have a single node Mesos cluster running and a single node +HDFS cluster running. You can modify the Mesos and Hadoop versions by editing the +corresponding variables in the `Vagrantfile`. + +Check that Mesos and Hadoop are running with a browser: + +[Hadoop/HDFS namenode](http://10.141.141.20:50070) +[Mesos Master](http://10.141.141.20:5050/) To ssh in the cluster, run following: @@ -23,51 +30,15 @@ To ssh in the cluster, run following: vagrant ssh ``` -The password for vagrant user is **vagrant'** - -## Setting Up YARN/Hadoop ## - -To setup YARN/Hadoop inside VM, run following YARN setup shell files: - -1 Run the first YARN setup shell command from the vagrant directory to create a user hduser in group hadoop. Be sure to remember the password that you provide for this user. -``` -cd /vagrant -./setup-yarn-1.sh -``` -2. Run the second YARN setup shell command as sudo. +The password for vagrant user is **vagrant** if prompted +You can switch to the hadoop user when needed to launch hadoop processes ``` sudo su - hduser -cd /vagrant -./setup-yarn-2.sh -``` - -The following processes should be running: - -``` -9844 Jps -6709 NameNode -6393 JobHistoryServer -6874 DataNode ``` -**NOTE:** The process IDS are different. - -## Installing Myriad ## - -Installing Myriad involves downloading the Myriad binaries from GitHub. The Myriad location is currently at [https://github.com/mesos/myriad](https://github.com/mesos/myriad). - -To install Myriad: - -1. Create a new myriad directory. -2. Clone the Myriad GitHub. -3. Pull the contents to the new directory. - - ## Configuring Myriad ## -To configure Myriad: - ### Step 1: Build Myriad ### To build Myriad Scheduler inside VM, run the gradlew build: @@ -214,7 +185,7 @@ To shut down, from the vagrant ssh console, run the following: ``` yarn-daemon.sh stop resourcemanager -./shutdown.sh +sh ./vagrant/shutdown.sh exit exit vagrant halt http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/myriad-scheduler/src/main/resources/webapp/.npmrc ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/resources/webapp/.npmrc b/myriad-scheduler/src/main/resources/webapp/.npmrc new file mode 100644 index 0000000..d86ebdf --- /dev/null +++ b/myriad-scheduler/src/main/resources/webapp/.npmrc @@ -0,0 +1 @@ +bin-links=false http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/setup-yarn-1.sh ---------------------------------------------------------------------- diff --git a/setup-yarn-1.sh b/setup-yarn-1.sh deleted file mode 100755 index 314d5cc..0000000 --- a/setup-yarn-1.sh +++ /dev/null @@ -1,18 +0,0 @@ -set -e - -HADOOP_VER=2.7.0 - -cd ~/ - -sudo apt-get update - -sudo apt-get install openssh-server - -# Add hduser user -sudo addgroup hadoop -sudo adduser --ingroup hadoop hduser -sudo adduser hduser sudo - -# Setup password-less auth -sudo -u hduser ssh-keygen -t rsa -P '' -sudo sh -c 'cat /home/hduser/.ssh/id_rsa.pub >> /home/hduser/.ssh/authorized_keys' http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/setup-yarn-2.sh ---------------------------------------------------------------------- diff --git a/setup-yarn-2.sh b/setup-yarn-2.sh deleted file mode 100755 index 074f531..0000000 --- a/setup-yarn-2.sh +++ /dev/null @@ -1,64 +0,0 @@ -set -e - -HADOOP_VER=2.7.0 - -# Download Hadoop -cd ~ -if [ ! -f hadoop-${HADOOP_VER}.tar.gz ]; then - wget http://apache.osuosl.org/hadoop/common/hadoop-${HADOOP_VER}/hadoop-${HADOOP_VER}.tar.gz -fi - -sudo tar vxzf hadoop-${HADOOP_VER}.tar.gz -C /usr/local -cd /usr/local -sudo mv hadoop-${HADOOP_VER} hadoop -sudo chown -R hduser:hadoop hadoop - -# Init bashrc with hadoop env variables -sudo sh -c 'echo export JAVA_HOME=/usr/lib/jvm/java-8-oracle >> /home/hduser/.bashrc' -sudo sh -c 'echo export HADOOP_INSTALL=/usr/local/hadoop >> /home/hduser/.bashrc' -sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/bin >> /home/hduser/.bashrc' -sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/sbin >> /home/hduser/.bashrc' -sudo sh -c 'echo export HADOOP_MAPRED_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc' -sudo sh -c 'echo export HADOOP_COMMON_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc' -sudo sh -c 'echo export HADOOP_HDFS_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc' -sudo sh -c 'echo export YARN_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc' -sudo sh -c 'echo export HADOOP_COMMON_LIB_NATIVE_DIR=\$\{HADOOP_INSTALL\}/lib/native >> /home/hduser/.bashrc' -sudo sh -c 'echo export HADOOP_OPTS=\"-Djava.library.path=\$HADOOP_INSTALL/lib\" >> /home/hduser/.bashrc' - -# Modify JAVA_HOME in hadoop-env -cd /usr/local/hadoop/etc/hadoop -sudo -u hduser sed -i.bak s=\${JAVA_HOME}=/usr/lib/jvm/java-8-oracle/=g hadoop-env.sh -pwd - -/usr/local/hadoop/bin/hadoop version - -# Update configuration -sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>fs\.default\.name\</name>\<value>hdfs://localhost:9000\</value>\</property>=g' core-site.xml -sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>yarn\.nodemanager\.aux-services</name>\<value>mapreduce_shuffle</value>\</property>\<property>\<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>\<value>org\.apache\.hadoop\.mapred\.ShuffleHandler</value>\</property>=g' yarn-site.xml - -sudo -u hduser cp mapred-site.xml.template mapred-site.xml -sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>mapreduce\.framework\.name</name>\<value>yarn</value>\</property>=g' mapred-site.xml - -cd ~ -sudo -u hduser sh -c 'mkdir -p mydata/hdfs/namenode' -sudo -u hduser sh -c 'mkdir -p mydata/hdfs/datanode' -sudo chown -R hduser:hadoop /home/hduser/mydata - -cd /usr/local/hadoop/etc/hadoop -sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>dfs\.replication</name>\<value>1\</value>\</property>\<property>\<name>dfs\.namenode\.name\.dir</name>\<value>file:/home/hduser/mydata/hdfs/namenode</value>\</property>\<property>\<name>dfs\.datanode\.data\.dir</name>\<value>file:/home/hduser/mydata/hdfs/datanode</value>\</property>=g' hdfs-site.xml - - -# Format NameNode -sudo -u hduser sh -c '/usr/local/hadoop/bin/hdfs namenode -format' - -# Start NameNode -sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemon.sh start namenode' - -# Start DataNode -sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemons.sh start datanode' - -# Start HistoryServer -sudo -u hduser sh -c '/usr/local/hadoop/sbin/mr-jobhistory-daemon.sh start historyserver' - -# Check status -sudo -u hduser -u hduser jps http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/shutdown.sh ---------------------------------------------------------------------- diff --git a/shutdown.sh b/shutdown.sh deleted file mode 100755 index b0bfab5..0000000 --- a/shutdown.sh +++ /dev/null @@ -1,10 +0,0 @@ -set -e - -# Stop HistoryServer -sudo -u hduser sh -c '/usr/local/hadoop/sbin/mr-jobhistory-daemon.sh stop historyserver' - -# Stop DataNode -sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemons.sh stop datanode' - -# Stop NameNode -sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemon.sh stop namenode' http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/format_namenode.sh ---------------------------------------------------------------------- diff --git a/vagrant/format_namenode.sh b/vagrant/format_namenode.sh new file mode 100644 index 0000000..e944a41 --- /dev/null +++ b/vagrant/format_namenode.sh @@ -0,0 +1,23 @@ +#!/bin/bash -v +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +set -e + +# Format NameNode +sudo -u hduser sh -c 'yes Y | /usr/local/hadoop/bin/hdfs namenode -format' http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/install_default_jdk.sh ---------------------------------------------------------------------- diff --git a/vagrant/install_default_jdk.sh b/vagrant/install_default_jdk.sh new file mode 100644 index 0000000..0a85c3c --- /dev/null +++ b/vagrant/install_default_jdk.sh @@ -0,0 +1,32 @@ +#!/bin/bash -v +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +PREFIX="JAVA Default JDK Provisioner:" +set -e + +# For installing Java 8 +apt-get -y update +apt-get -y install default-jdk + +if $(test -e /usr/lib/libjvm.so); then + rm /usr/lib/libjvm.so +fi + +ln -s /usr/lib/jvm/default-java/jre/lib/amd64/server/libjvm.so /usr/lib/libjvm.so http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/install_docker.sh ---------------------------------------------------------------------- diff --git a/vagrant/install_docker.sh b/vagrant/install_docker.sh new file mode 100644 index 0000000..a0a214f --- /dev/null +++ b/vagrant/install_docker.sh @@ -0,0 +1,25 @@ +#!/bin/bash -v +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +set -e + +#Install docker +echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list +apt-get update +apt-get -y install docker.io http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/install_hadoop.sh ---------------------------------------------------------------------- diff --git a/vagrant/install_hadoop.sh b/vagrant/install_hadoop.sh new file mode 100644 index 0000000..fe5c2a0 --- /dev/null +++ b/vagrant/install_hadoop.sh @@ -0,0 +1,115 @@ +#!/bin/bash -v +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# $1 is HADOOP_VERSION +set -e + +apt-get update + +apt-get install -y openssh-server +apt-get install -y tar +apt-get install -y gzip + +# Add hduser user and hadoop group + +if [ `/bin/egrep -i "^hadoop:" /etc/group` ]; then + echo "Group hadoop already exists" +else + echo "Adding hadoop group" + addgroup hadoop +fi + + +if [ `/bin/egrep -i "^hduser:" /etc/passwd` ]; then + echo "User hduser already exists" +else + echo "creating hduser in group hadoop" + adduser --ingroup hadoop --disabled-password --gecos "" --home /home/hduser hduser + adduser hduser sudo +fi + +# Setup password-less auth +sudo -u hduser sh -c "mkdir -p /home/hduser/.ssh" +sudo -u hduser sh -c "chmod 700 /home/hduser/.ssh" +sudo -u hduser sh -c "yes | ssh-keygen -t rsa -N '' -f /home/hduser/.ssh/id_rsa" +sudo -u hduser sh -c 'cat /home/hduser/.ssh/id_rsa.pub >> /home/hduser/.ssh/authorized_keys' +#sudo -u hduser sh -c "ssh-keyscan -H $1 >> /home/hduser/.ssh/known_hosts" +sudo -u hduser sh -c "ssh-keyscan -H localhost >> /home/hduser/.ssh/known_hosts" +#sudo -u hduser sh -c "ssh-keyscan -H $2 >> /home/hduser/.ssh/known_hosts" + +# Download Hadoop +HADOOP_VER="$1" +cd ~ +if [ ! -f /tmp/hadoop-${HADOOP_VER}.tar.gz ]; then + wget http://apache.osuosl.org/hadoop/common/hadoop-${HADOOP_VER}/hadoop-${HADOOP_VER}.tar.gz -O /tmp/hadoop-${HADOOP_VER}.tar.gz +fi + +sudo tar ixzf /tmp/hadoop-${HADOOP_VER}.tar.gz -C /usr/local +cd /usr/local +rm -rf hadoop +sudo mv -f hadoop-${HADOOP_VER} hadoop +sudo chown -R hduser:hadoop hadoop + +# Init bashrc with hadoop env variables +sudo sh -c 'echo export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 >> /home/hduser/.bashrc' +sudo sh -c 'echo export HADOOP_INSTALL=/usr/local/hadoop >> /home/hduser/.bashrc' +sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/bin >> /home/hduser/.bashrc' +sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/sbin >> /home/hduser/.bashrc' +sudo sh -c 'echo export HADOOP_MAPRED_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc' +sudo sh -c 'echo export HADOOP_COMMON_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc' +sudo sh -c 'echo export HADOOP_HDFS_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc' +sudo sh -c 'echo export YARN_HOME=\$HADOOP_INSTALL >> /home/hduser/.bashrc' +sudo sh -c 'echo export HADOOP_COMMON_LIB_NATIVE_DIR=\$\{HADOOP_INSTALL\}/lib/native >> /home/hduser/.bashrc' +sudo sh -c 'echo export HADOOP_OPTS=\"-Djava.library.path=\$HADOOP_INSTALL/lib\" >> /home/hduser/.bashrc' +# hit the vagrant user with the same thing +sudo sh -c 'echo export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 >> /home/vagrant/.bashrc' +sudo sh -c 'echo export HADOOP_INSTALL=/usr/local/hadoop >> /home/vagrant/.bashrc' +sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/bin >> /home/vagrant/.bashrc' +sudo sh -c 'echo export PATH=\$PATH:\$HADOOP_INSTALL/sbin >> /home/vagrant/.bashrc' +sudo sh -c 'echo export HADOOP_MAPRED_HOME=\$HADOOP_INSTALL >> /home/vagrant/.bashrc' +sudo sh -c 'echo export HADOOP_COMMON_HOME=\$HADOOP_INSTALL >> /home/vagrant/.bashrc' +sudo sh -c 'echo export HADOOP_HDFS_HOME=\$HADOOP_INSTALL >> /home/vagrant/.bashrc' +sudo sh -c 'echo export YARN_HOME=\$HADOOP_INSTALL >> /home/vagrant/.bashrc' +sudo sh -c 'echo export HADOOP_COMMON_LIB_NATIVE_DIR=\$\{HADOOP_INSTALL\}/lib/native >> /home/vagrant/.bashrc' +sudo sh -c 'echo export HADOOP_OPTS=\"-Djava.library.path=\$HADOOP_INSTALL/lib\" >> /home/vagrant/.bashrc' + + +# Modify JAVA_HOME in hadoop-env +cd /usr/local/hadoop/etc/hadoop +sudo -u hduser sed -i.bak s=\${JAVA_HOME}=//usr/lib/jvm/java-7-openjdk-amd64/=g hadoop-env.sh +pwd + +/usr/local/hadoop/bin/hadoop version + +# Update configuration +sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>fs\.default\.name\</name>\<value>hdfs://localhost:9000\</value>\</property>=g' core-site.xml +#sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>fs\.default\.name\</name>\<value>hdfs://'"$1"':9000\</value>\</property>=g' core-site.xml +sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>yarn\.nodemanager\.aux-services</name>\<value>mapreduce_shuffle</value>\</property>\<property>\<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>\<value>org\.apache\.hadoop\.mapred\.ShuffleHandler</value>\</property>=g' yarn-site.xml + +sudo -u hduser cp mapred-site.xml.template mapred-site.xml +sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>mapreduce\.framework\.name</name>\<value>yarn</value>\</property>=g' mapred-site.xml + +cd ~ +sudo -u hduser sh -c 'mkdir -p ~hduser/mydata/hdfs/namenode' +sudo -u hduser sh -c 'mkdir -p ~hduser/mydata/hdfs/datanode' +sudo chown -R hduser:hadoop ~hduser/mydata + +cd /usr/local/hadoop/etc/hadoop +sudo -u hduser sed -i.bak 's=<configuration>=<configuration>\<property>\<name>dfs\.replication</name>\<value>1\</value>\</property>\<property>\<name>dfs\.namenode\.name\.dir</name>\<value>file:/home/hduser/mydata/hdfs/namenode</value>\</property>\<property>\<name>dfs\.datanode\.data\.dir</name>\<value>file:/home/hduser/mydata/hdfs/datanode</value>\</property>\<property>\<name>dfs\.namenode\.datanode\.registration\.ip-hostname-check</name>\<value>false</value>\</property>=g' hdfs-site.xml http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/install_mesos.sh ---------------------------------------------------------------------- diff --git a/vagrant/install_mesos.sh b/vagrant/install_mesos.sh new file mode 100644 index 0000000..28292bf --- /dev/null +++ b/vagrant/install_mesos.sh @@ -0,0 +1,43 @@ +#!/bin/bash -v +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# $1 = mesos version number to install (e.g. 0.21.1) + +PREFIX="Mesos Provisioner: " +set -e + +echo "${PREFIX} Installing pre-reqs..." +# For Mesos +apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF +DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') +CODENAME=$(lsb_release -cs) +echo "deb http://repos.mesosphere.io/${DISTRO} ${CODENAME} main" | sudo tee /etc/apt/sources.list.d/mesosphere.list +apt-get -y update + +apt-get -y install libcurl3 +apt-get -y install zookeeperd +apt-get -y install aria2 +apt-get -y install ssh +apt-get -y install rsync + + +export MESOS_VERSION=$1 +echo "${PREFIX}Installing mesos version: ${MESOS_VERSION}..." +apt-get -y install mesos http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/shutdown.sh ---------------------------------------------------------------------- diff --git a/vagrant/shutdown.sh b/vagrant/shutdown.sh new file mode 100644 index 0000000..2012b84 --- /dev/null +++ b/vagrant/shutdown.sh @@ -0,0 +1,30 @@ +#!/bin/bash -v +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +# Stop HistoryServer +sudo -u hduser sh -c '/usr/local/hadoop/sbin/mr-jobhistory-daemon.sh stop historyserver' + +# Stop DataNode +sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemons.sh stop datanode' + +# Stop NameNode +sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemon.sh stop namenode' http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/start_datanode.sh ---------------------------------------------------------------------- diff --git a/vagrant/start_datanode.sh b/vagrant/start_datanode.sh new file mode 100644 index 0000000..e391187 --- /dev/null +++ b/vagrant/start_datanode.sh @@ -0,0 +1,24 @@ +#!/bin/bash -v +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +# Start DataNode +sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemons.sh start datanode' http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/start_historyserver.sh ---------------------------------------------------------------------- diff --git a/vagrant/start_historyserver.sh b/vagrant/start_historyserver.sh new file mode 100644 index 0000000..88adfca --- /dev/null +++ b/vagrant/start_historyserver.sh @@ -0,0 +1,23 @@ +#!/bin/bash -v +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +sudo -u hduser sh -c '/usr/local/hadoop/sbin/mr-jobhistory-daemon.sh start historyserver' http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/start_mesos_master.sh ---------------------------------------------------------------------- diff --git a/vagrant/start_mesos_master.sh b/vagrant/start_mesos_master.sh new file mode 100644 index 0000000..ebe9e66 --- /dev/null +++ b/vagrant/start_mesos_master.sh @@ -0,0 +1,34 @@ +#!/bin/bash -v +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +echo "zk://127.0.0.1:2181/mesos" > /etc/mesos/zk +#echo "$@ master" >> /etc/hosts +#echo "192.168.50.102 slave" >> /etc/hosts +echo 127.0.0.1 | sudo tee /etc/mesos-master/ip +echo localhost | sudo tee /etc/mesos-master/hostname + +echo "export HADOOP_HOME=/usr/local/hadoop" >> /root/.bashrc +echo "export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64" >> /root/.bashrc +echo "export PATH=$PATH:$HADOOP_HOME/bin:$JAVA_HOME/bin" >> /root/.bashrc + +# keep mesos slave from starting here +echo manual | sudo tee /etc/init/mesos-slave.override + +sudo start mesos-master http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/start_mesos_slave.sh ---------------------------------------------------------------------- diff --git a/vagrant/start_mesos_slave.sh b/vagrant/start_mesos_slave.sh new file mode 100644 index 0000000..eeca9e1 --- /dev/null +++ b/vagrant/start_mesos_slave.sh @@ -0,0 +1,38 @@ +#!/bin/bash -v +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +echo "127.0.0.1" > /etc/mesos-slave/ip +echo "cgroups/cpu,cgroups/mem" > /etc/mesos-slave/isolation +echo "mesos" > /etc/mesos-slave/containerizers +echo "/usr/local/hadoop" > /etc/mesos-slave/hadoop_home + +echo "export HADOOP_HOME=/usr/local/hadoop" >> /root/.bashrc +echo "export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64" >> /root/.bashrc + +echo "zk://127.0.0.1:2181/mesos" | sudo tee /etc/mesos/zk +echo "localhost" | sudo tee /etc/mesos-slave/hostname +echo "cpus:2;mem:2048" | sudo tee /etc/mesos-slave/resources +echo manual | sudo tee /etc/init/mesos-master.override +echo manual | sudo tee /etc/init/zookeeper.override + + +start mesos-slave http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/fed956b8/vagrant/start_namenode.sh ---------------------------------------------------------------------- diff --git a/vagrant/start_namenode.sh b/vagrant/start_namenode.sh new file mode 100644 index 0000000..d82b260 --- /dev/null +++ b/vagrant/start_namenode.sh @@ -0,0 +1,27 @@ +#!/bin/bash -v +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e + +# Start NameNode +sudo -u hduser sh -c '/usr/local/hadoop/sbin/hadoop-daemon.sh start namenode' + +# Open up the HDFS permissions +sudo -u hduser sh -c "/usr/local/hadoop/bin/hadoop fs -chmod 777 /"