Repository: bigtop Updated Branches: refs/heads/master 263af453a -> 3d4ed8c1e
BIGTOP-2646. Move Vagrant Provisioner from bigtop-deploy/vm to provisioner directory Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/3d4ed8c1 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/3d4ed8c1 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/3d4ed8c1 Branch: refs/heads/master Commit: 3d4ed8c1ee278e553ae052a82bcecb3f189dd2f1 Parents: 263af45 Author: Evans Ye <[email protected]> Authored: Wed Dec 28 01:31:54 2016 +0800 Committer: Evans Ye <[email protected]> Committed: Sun Jan 8 17:01:12 2017 +0800 ---------------------------------------------------------------------- provisioner/utils/setup-env-centos.sh | 40 +++++++ provisioner/utils/setup-env-debian.sh | 33 ++++++ provisioner/utils/smoke-tests.sh | 64 +++++++++++ provisioner/vagrant/.gitignore | 1 + provisioner/vagrant/README.md | 99 +++++++++++++++++ provisioner/vagrant/Vagrantfile | 166 ++++++++++++++++++++++++++++ provisioner/vagrant/config_hosts | 45 ++++++++ provisioner/vagrant/gen_hosts.sh | 44 ++++++++ provisioner/vagrant/vagrantconfig.yaml | 26 +++++ 9 files changed, 518 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/3d4ed8c1/provisioner/utils/setup-env-centos.sh ---------------------------------------------------------------------- diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh new file mode 100755 index 0000000..86b6a3c --- /dev/null +++ b/provisioner/utils/setup-env-centos.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable_local_repo=${1:-false} + +# Setup rng-tools to improve virtual machine entropy performance. +# The poor entropy performance will cause kerberos provisioning failed. +yum -y install rng-tools +if [ -x /usr/bin/systemctl ] ; then + sed -i 's@ExecStart=/sbin/rngd -f@ExecStart=/sbin/rngd -f -r /dev/urandom@' /usr/lib/systemd/system/rngd.service + systemctl daemon-reload + systemctl start rngd +else + sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd + service rngd start +fi + +if [ $enable_local_repo == "true" ]; then + echo "Enabling local yum." + yum -y install yum-utils + sudo echo "gpgcheck=0" >> /etc/yum.conf + sudo yum-config-manager --add-repo file:///bigtop-home/output +else + echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..." +fi + http://git-wip-us.apache.org/repos/asf/bigtop/blob/3d4ed8c1/provisioner/utils/setup-env-debian.sh ---------------------------------------------------------------------- diff --git a/provisioner/utils/setup-env-debian.sh b/provisioner/utils/setup-env-debian.sh new file mode 100755 index 0000000..098155b --- /dev/null +++ b/provisioner/utils/setup-env-debian.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +enable_local_repo=${1:-false} + +# Setup rng-tools to improve virtual machine entropy performance. +# The poor entropy performance will cause kerberos provisioning failed. +apt-get -y install rng-tools +sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools +service rng-tools start + +if [ $enable_local_repo == "true" ]; then + echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list + apt-get update +else + apt-get install -y apt-transport-https + echo "local apt = $enable_local_repo ; NOT Enabling local apt. Packages will be pulled from remote..." +fi + http://git-wip-us.apache.org/repos/asf/bigtop/blob/3d4ed8c1/provisioner/utils/smoke-tests.sh ---------------------------------------------------------------------- diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh new file mode 100755 index 0000000..93f795e --- /dev/null +++ b/provisioner/utils/smoke-tests.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HCFS_USER="hdfs" +SMOKE_TESTS=${1:-mapreduce,pig} + +# Autodetect JAVA_HOME +if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then + . /usr/lib/bigtop-utils/bigtop-detect-javahome +else + >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n" + exit 2 +fi + +echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n" + +export HADOOP_CONF_DIR=/etc/hadoop/conf/ +export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/ +export HIVE_HOME=/usr/lib/hive/ +export PIG_HOME=/usr/lib/pig/ +export FLUME_HOME=/usr/lib/flume/ +export SQOOP_HOME=/usr/lib/sqoop/ +export HIVE_CONF_DIR=/etc/hive/conf/ +export MAHOUT_HOME="/usr/lib/mahout" + +prep() { + HADOOP_COMMAND=$1 + su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -mkdir /user/vagrant /user/root" + su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/vagrant" + su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/root" +} + +prep hadoop +if [[ $SMOKE_TESTS == *"qfs"* ]]; then + prep hadoop-qfs +fi + +if [ -f /etc/debian_version ] ; then + apt-get -y install pig hive flume mahout sqoop +else + yum install -y pig hive flume mahout sqoop +fi +ALL_SMOKE_TASKS="" +for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do + ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test" +done +cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info +# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same +# permissions as used for smoke-tests execution +rm -rf buildSrc/build/test-results/binary + http://git-wip-us.apache.org/repos/asf/bigtop/blob/3d4ed8c1/provisioner/vagrant/.gitignore ---------------------------------------------------------------------- diff --git a/provisioner/vagrant/.gitignore b/provisioner/vagrant/.gitignore new file mode 100644 index 0000000..04204c7 --- /dev/null +++ b/provisioner/vagrant/.gitignore @@ -0,0 +1 @@ +config http://git-wip-us.apache.org/repos/asf/bigtop/blob/3d4ed8c1/provisioner/vagrant/README.md ---------------------------------------------------------------------- diff --git a/provisioner/vagrant/README.md b/provisioner/vagrant/README.md new file mode 100644 index 0000000..0b1ff20 --- /dev/null +++ b/provisioner/vagrant/README.md @@ -0,0 +1,99 @@ + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------------------------------------------------------------------------ + +#BigTop VM provisioner + +## Overview + +The Vagrantfile definition creates a bigtop virtual hadoop cluster for you, by pulling from existing publishing bigtop repositories. +This cluster can be used: + +- to test bigtop smoke tests +- to test bigtop puppet recipes + +Eventually, we may also add ability to build AND provision bigtop in a vagrant recipe, which would essentially +give full validation of the BigTop stack. + +## USAGE + +1) Install [Vagrant](https://www.vagrantup.com/downloads.html)(DO NOT install 1.8.5 because of a critical [bug](https://github.com/mitchellh/vagrant/issues/7631)) + +2) Install [vagrant-hostmanager plugin](https://github.com/smdahlen/vagrant-hostmanager) to manage `/etc/hosts` + +``` +vagrant plugin install vagrant-hostmanager +``` + +3) (Optional) Install [vagrant-cachier plugin](https://github.com/fgrehm/vagrant-cachier) to cache packages at local + +``` +vagrant plugin install vagrant-cachier +``` + +4) To provision a one node Apache Hadoop cluster on top of vagrant boxes + +``` +vagrant up +``` + +5) You can specify number of nodes you'd like to provision by modifying `num_instances` in vagrantconfig.yaml + +``` +num_instances: 5 +``` + +6) Test on local built packages is available by: + +first, build up local yum repo + +``` +cd bigtop; ./gradlew alluxio-yum +``` + +and then enable local yum in vagrantconfig.yaml + +``` +enable_local_repo: true +``` + +finally, spin up the cluster +``` +vagrant up +``` + +## Override Vagrant configurations +You can override vagrant configurations from environment variables: +``` +$ export REPO=http://repo.example.com/path/to +$ vagrant up +``` + +or + +``` +$ MEMORY_SIZE=8000 vagrant up +``` + +##Configure Apache Hadoop ecosystem components +* Choose the ecosystem you want to be deployed by modifying components in vagrantconfig.yaml + +``` +components: "hadoop, hbase, yarn,..." +``` + +By default, Apache Hadoop and YARN will be installed. +See `bigtop-deploy/puppet/hieradata/site.yaml` for more details. http://git-wip-us.apache.org/repos/asf/bigtop/blob/3d4ed8c1/provisioner/vagrant/Vagrantfile ---------------------------------------------------------------------- diff --git a/provisioner/vagrant/Vagrantfile b/provisioner/vagrant/Vagrantfile new file mode 100755 index 0000000..fc6bfc1 --- /dev/null +++ b/provisioner/vagrant/Vagrantfile @@ -0,0 +1,166 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "yaml" + +_config = YAML.load(File.open(File.join(File.dirname(__FILE__), "vagrantconfig.yaml"), File::RDONLY).read) +CONF = _config + +# Override vagrant configurations using environment variables +keys = CONF.keys +keys.each do |k| + if ENV[k.upcase] != nil then + puts "Overide from environment variable: " + k.upcase + " = " + ENV[k.upcase] + if /^\d+/.match(ENV[k.upcase]) + CONF[k] = Integer(ENV[k.upcase]) + else + CONF[k] = ENV[k.upcase] + end + end +end + +# Repository +# Example for testing a Release candidate. +repo = CONF['repo'] +# repo = "http://bigtop.s3.amazonaws.com/releases/0.7.0/redhat/6/x86_64"; + +# Which Linux Distribution to use. Right now only centos is tested +distro = CONF['distro'] + +# number of instances +num_instances = CONF['num_instances'] + +# hadoop ecosystem components +components = CONF['components'] + +# Whether to run smoke tests +run_smoke_tests = CONF['run_smoke_tests'] + +# Smoke test Components to run +smoke_test_components = CONF['smoke_test_components'].join(',') + +# This is a update to allow dev packages +# Force success - not worried if this step fails, since we generally only use it for development. +enable_local_repo = CONF['enable_local_repo'] +puts "vagrant conf local repo enabled: #{enable_local_repo}" + +# JDK package name +jdk = CONF['jdk'] + +# master node hostname +bigtop_master = "bigtop1.vagrant" + +$script = <<SCRIPT +service iptables stop +service firewalld stop +chkconfig iptables off +# Remove 127.0.0.1 entry since vagrant's hostname setting will map it to FQDN, +# which miss leads some daemons to bind on 127.0.0.1 instead of public or private IP +sed -i "/127.0.0.1/d" /etc/hosts +echo "Bigtop yum repo = #{repo}" +# Prepare host manipulation files needed in standalone box +cp /bigtop-home/provisioner/vagrant/config_hosts /etc/init.d +cp /bigtop-home/provisioner/vagrant/gen_hosts.sh /usr/bin +chkconfig --add config_hosts +# Prepare puppet configuration file +mkdir -p /etc/puppet/hieradata +cp /bigtop-home/bigtop-deploy/puppet/hiera.yaml /etc/puppet +cp -r /bigtop-home/bigtop-deploy/puppet/hieradata/bigtop/ /etc/puppet/hieradata/ +cat > /etc/puppet/hieradata/site.yaml << EOF +bigtop::hadoop_head_node: #{bigtop_master} +hadoop::hadoop_storage_dirs: [/data/1, /data/2] +bigtop::bigtop_repo_uri: #{repo} +hadoop_cluster_node::cluster_components: #{components} +bigtop::jdk_package_name: #{jdk} +hadoop::common_hdfs::testonly_hdfs_sshkeys: "yes" +EOF +SCRIPT + +VAGRANTFILE_API_VERSION = "2" +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + # manage /etc/hosts by hostmanager plugin(https://github.com/smdahlen/vagrant-hostmanager) + # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier) + config.hostmanager.enabled = true + + # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier) + if Vagrant.has_plugin?("vagrant-cachier") + config.cache.scope = :box + end + + # nodes definition + (1..num_instances).each do |i| + config.vm.define "bigtop#{i}" do |bigtop| + + bigtop.vm.box = CONF['box'] + bigtop_hostname="bigtop#{i}" + bigtop_fqdn="#{bigtop_hostname}.vagrant" + bigtop_ip="10.10.10.1#{i}" + + bigtop.vm.provider :virtualbox do |vb| + vb.customize ["modifyvm", :id, "--memory", CONF['memory_size']] + vb.customize ['modifyvm', :id, '--cpus', CONF['number_cpus']] + end + + bigtop.vm.network :private_network, ip: bigtop_ip + bigtop.vm.hostname = bigtop_fqdn + + # two levels up is the bigtop "home" directory. + # the current directory has puppet recipes which we need for provisioning. + bigtop.vm.synced_folder "../../", "/bigtop-home" + + # We also add the bigtop-home output/ dir, so that locally built rpms will be available. + puts "Adding output/ repo ? #{enable_local_repo}" + + # Install puppet + bigtop.vm.provision :shell do |shell| + shell.path = "../../bigtop_toolchain/bin/puppetize.sh" + end + + # carry on w/ installation + bigtop.vm.provision :shell do |shell| + shell.path = "../utils/setup-env-" + distro + ".sh" + shell.args = ["#{enable_local_repo}"] + end + bigtop.vm.provision "shell", inline: $script + # Add the ip to FQDN and hostname mapping in /etc/hosts + bigtop.hostmanager.aliases = "#{bigtop_fqdn} #{bigtop_hostname}" + + # deploy Apache Hadoop and it's ecosystem + bigtop.vm.provision :puppet do |puppet| + puppet.module_path = "../../bigtop-deploy/puppet/modules/" + puppet.manifests_path = "../../bigtop-deploy/puppet/manifests/" + puppet.manifest_file = "." + puppet.options = "--parser future" + end + + if run_smoke_tests then + if i==num_instances then + puts "creating provisioner directive for running tests" + bigtop.vm.provision :shell do |shell| + shell.path = "../utils/smoke-tests.sh" + shell.args = ["#{smoke_test_components}"] + end + else + puts "Not creating provisioner directive for tests yet... only on vm #{i} of #{num_instances}" + end + end + + end + end + +end http://git-wip-us.apache.org/repos/asf/bigtop/blob/3d4ed8c1/provisioner/vagrant/config_hosts ---------------------------------------------------------------------- diff --git a/provisioner/vagrant/config_hosts b/provisioner/vagrant/config_hosts new file mode 100755 index 0000000..07e1d5f --- /dev/null +++ b/provisioner/vagrant/config_hosts @@ -0,0 +1,45 @@ +#!/bin/bash + +# Hosts reconfiguration script +# chkconfig: 345 70 1 +# description: config_hosts + +# Licensed to Hortonworks, Inc. under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Hortonworks, Inc. licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +NAME="config_hosts" +LOG="/var/log/config_hosts.log" + +source /etc/rc.d/init.d/functions + +case "$1" in +start) + printf "%-50s\n" "Starting $NAME..." | tee -a $LOG + printf "%-50s\n" "Updating IP..." | tee -a $LOG + bash /usr/bin/gen_hosts.sh +;; + +stop) + date >> $LOG +;; + +restart) +;; + +*) + echo "Usage: $0 {start|stop|restart}" + exit 1 +esac http://git-wip-us.apache.org/repos/asf/bigtop/blob/3d4ed8c1/provisioner/vagrant/gen_hosts.sh ---------------------------------------------------------------------- diff --git a/provisioner/vagrant/gen_hosts.sh b/provisioner/vagrant/gen_hosts.sh new file mode 100755 index 0000000..c629c85 --- /dev/null +++ b/provisioner/vagrant/gen_hosts.sh @@ -0,0 +1,44 @@ +# Licensed to Hortonworks, Inc. under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Hortonworks, Inc. licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "# Do not remove the following line, or various programs" > /etc/hosts +echo "# that require network functionality will fail." >> /etc/hosts +echo "127.0.0.1 localhost.localdomain localhost" >> /etc/hosts + +function get_inet_iface(){ + route | grep default | awk '{print $8}' +} + +function get_ip() { + ip addr | grep 'inet ' | grep -E "($(get_inet_iface))" | awk '{ print $2 }' | awk -F'/' '{print $1}' +} + +HOST=$(get_ip) +NUM=5 +while [ -z "$HOST" ]; do + HOST=$(get_ip) + sleep 5 + NUM=$(($NUM-1)) + if [ $NUM -le 0 ]; then + HOST="127.0.0.1" + echo "Failed to update IP" + break + fi +done +echo "$HOST `hostname`" >> /etc/hosts + +echo 0 > /proc/sys/kernel/hung_task_timeout_secs +ethtool -K $(get_inet_iface) tso off http://git-wip-us.apache.org/repos/asf/bigtop/blob/3d4ed8c1/provisioner/vagrant/vagrantconfig.yaml ---------------------------------------------------------------------- diff --git a/provisioner/vagrant/vagrantconfig.yaml b/provisioner/vagrant/vagrantconfig.yaml new file mode 100644 index 0000000..4ad1aac --- /dev/null +++ b/provisioner/vagrant/vagrantconfig.yaml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +memory_size: 4096 +number_cpus: 1 +box: "puppetlabs/centos-7.2-64-nocm" +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64"; +num_instances: 1 +distro: centos +components: [hadoop, yarn, hbase] +enable_local_repo: false +run_smoke_tests: false +smoke_test_components: [mapreduce, pig] +jdk: "java-1.7.0-openjdk-devel.x86_64"
