Repository: bigtop Updated Branches: refs/heads/master 95ff0128a -> 801bade70
BIGTOP-1517. refactor vagrant provisioners to be configurable by yaml file Signed-off-by: jayunit100 <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/801bade7 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/801bade7 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/801bade7 Branch: refs/heads/master Commit: 801bade70dd1634b2a9d67c8ba91b3260848e22d Parents: 95ff012 Author: evans_ye <[email protected]> Authored: Sun Nov 16 22:55:43 2014 +0800 Committer: jayunit100 <[email protected]> Committed: Wed Nov 19 18:17:21 2014 -0500 ---------------------------------------------------------------------- bigtop-deploy/vm/docker-puppet/README.md | 35 +++++++++++++++++-- bigtop-deploy/vm/docker-puppet/Vagrantfile | 13 ++++--- .../vm/docker-puppet/boot2docker/Vagrantfile | 14 +++++--- bigtop-deploy/vm/docker-puppet/docker-hadoop.sh | 4 ++- bigtop-deploy/vm/docker-puppet/provision.sh | 15 -------- .../vm/docker-puppet/vagrantconfig.yaml | 11 ++++++ bigtop-deploy/vm/smoke-tests.sh | 20 ----------- bigtop-deploy/vm/utils/hbase-test.sh | 24 +++++++++++++ bigtop-deploy/vm/utils/setup-env.sh | 31 +++++++++++++++++ bigtop-deploy/vm/utils/smoke-tests.sh | 20 +++++++++++ bigtop-deploy/vm/vagrant-puppet/README.md | 20 +++++------ bigtop-deploy/vm/vagrant-puppet/Vagrantfile | 36 ++++++++++---------- bigtop-deploy/vm/vagrant-puppet/hbase-test.sh | 24 ------------- bigtop-deploy/vm/vagrant-puppet/provision.sh | 17 ++------- .../vm/vagrant-puppet/vagrantconfig.yaml | 6 ++++ 15 files changed, 178 insertions(+), 112 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/docker-puppet/README.md ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/README.md b/bigtop-deploy/vm/docker-puppet/README.md index 2e8e1e2..c57a6b5 100644 --- a/bigtop-deploy/vm/docker-puppet/README.md +++ b/bigtop-deploy/vm/docker-puppet/README.md @@ -15,7 +15,7 @@ ------------------------------------------------------------------------------------------------------------------------------------------------------ -#BigTop docker provisioner +#BigTop Docker provisioner ## Overview @@ -115,7 +115,7 @@ build-image => create 5 node cluster => destroy the cluster 7) Run hbase-test.sh to evaluate the deployment. ``` -../vagrant-puppet/hbase-test.sh +../utils/hbase-test.sh ``` 8) See helper message: @@ -133,6 +133,37 @@ usage: docker-hadoop.sh [options] ``` +##Configurations + +* There are several parameters can be configured in the vagrantconfig.yaml: + +1) Modify memory limit for Docker containers + +``` +docker: + memory_size: "2048" + +``` + +2) If you're running Docker provisioner on OS X or Windows, you can customize the boot2docker VM settings + +``` +boot2docker: + memory_size: "4196" + number_cpus: "2" +``` + +3) Use different host ports mapping for web UIs + +``` +namenode_ui_port: "50070" +yarn_ui_port: "8088" +hbase_ui_port: "60010" + +``` +Note: If running on OS X or Windows, the boot2docker VM should be reloaded after ports changed + + ##Configure Apache Hadoop ecosystem components * Choose the ecosystem you want to be deployed by modifying components in provision.sh. http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/docker-puppet/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/Vagrantfile b/bigtop-deploy/vm/docker-puppet/Vagrantfile index fd73144..3ec023b 100755 --- a/bigtop-deploy/vm/docker-puppet/Vagrantfile +++ b/bigtop-deploy/vm/docker-puppet/Vagrantfile @@ -27,6 +27,11 @@ if File.exist?(CONFIG) require CONFIG end +require "yaml" + +_config = YAML.load(File.open(File.join(File.dirname(__FILE__), "vagrantconfig.yaml"), File::RDONLY).read) +CONF = _config + ENV['VAGRANT_DEFAULT_PROVIDER'] ||= 'docker' Vagrant.require_version ">= 1.6.0" @@ -38,7 +43,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.define "image", autostart: false do |image| image.vm.provider "docker" do |d| d.build_dir = "." - d.build_args = ["-t","bigtop/ssh:centos-6.4"] + d.build_args = ["-t",CONF["docker"]["image"]] d.vagrant_machine = "boot2docker" d.vagrant_vagrantfile = "./boot2docker/Vagrantfile" end @@ -51,10 +56,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.define "bigtop#{i}" do |bigtop| # docker container settings bigtop.vm.provider "docker" do |d| - d.image = "bigtop/ssh:centos-6.4" - d.create_args = ["--privileged=true"] + d.image = CONF["docker"]["image"] + d.create_args = ["--privileged=true", "-m", CONF["docker"]['memory_size'] + "m"] if "#{i}" == "1" - d.ports = ["50070:50070","8088:8088","60010:60010"] + d.ports = [CONF["namenode_ui_port"] + ":50070", CONF["yarn_ui_port"] + ":8088", CONF["hbase_ui_port"] + ":60010"] end d.vagrant_machine = "boot2docker" d.vagrant_vagrantfile = "./boot2docker/Vagrantfile" http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/docker-puppet/boot2docker/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/boot2docker/Vagrantfile b/bigtop-deploy/vm/docker-puppet/boot2docker/Vagrantfile index 8fa04f8..5913fec 100644 --- a/bigtop-deploy/vm/docker-puppet/boot2docker/Vagrantfile +++ b/bigtop-deploy/vm/docker-puppet/boot2docker/Vagrantfile @@ -1,6 +1,11 @@ # -*- mode: ruby -*- # vi: set ft=ruby : +require "yaml" + +_config = YAML.load(File.open(File.join(File.dirname(__FILE__), "../vagrantconfig.yaml"), File::RDONLY).read) +CONF = _config + VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| @@ -8,11 +13,12 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.define "boot2docker" do |node| node.vm.box = "yungsang/boot2docker" node.vm.provider "virtualbox" do |vb| - vb.customize ["modifyvm", :id, "--memory", "4196"] + vb.customize ["modifyvm", :id, "--memory", CONF["boot2docker"]["memory_size"]] + vb.customize ['modifyvm', :id, '--cpus', CONF["boot2docker"]['number_cpus']] end - node.vm.network "forwarded_port", host: 50070, guest: 50070 - node.vm.network "forwarded_port", host: 8088, guest: 8088 - node.vm.network "forwarded_port", host: 60010, guest: 60010 + node.vm.network "forwarded_port", host: CONF["namenode_ui_port"], guest: 50070 + node.vm.network "forwarded_port", host: CONF["yarn_ui_port"], guest: 8088 + node.vm.network "forwarded_port", host: CONF["hbase_ui_port"], guest: 60010 end end http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh b/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh index d48815c..21790ea 100755 --- a/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh +++ b/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh @@ -28,10 +28,12 @@ create() { vagrant up --no-parallel nodes=(`vagrant status |grep running |awk '{print $1}'`) hadoop_head_node=(`echo "hostname -f" |vagrant ssh ${nodes[0]} |tail -n 1`) + echo "/bigtop-home/bigtop-deploy/vm/utils/setup-env.sh" |vagrant ssh ${nodes[0]} echo "/vagrant/provision.sh $hadoop_head_node" |vagrant ssh ${nodes[0]} bigtop-puppet ${nodes[0]} for ((i=1 ; i<${#nodes[*]} ; i++)); do ( + echo "/bigtop-home/bigtop-deploy/vm/utils/setup-env.sh" |vagrant ssh ${nodes[$i]} echo "/vagrant/provision.sh $hadoop_head_node" |vagrant ssh ${nodes[$i]} bigtop-puppet ${nodes[$i]} ) & @@ -49,7 +51,7 @@ provision() { smoke-tests() { nodes=(`vagrant status |grep running |awk '{print $1}'`) - echo "/bigtop-home/bigtop-deploy/vm/smoke-tests.sh" |vagrant ssh ${nodes[0]} + echo "/bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh" |vagrant ssh ${nodes[0]} } http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/docker-puppet/provision.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/provision.sh b/bigtop-deploy/vm/docker-puppet/provision.sh index dfd58b6..46b2c01 100755 --- a/bigtop-deploy/vm/docker-puppet/provision.sh +++ b/bigtop-deploy/vm/docker-puppet/provision.sh @@ -15,10 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Install puppet agent -yum -y install http://yum.puppetlabs.com/puppetlabs-release-el-6.noarch.rpm -yum -y install puppet - sysctl kernel.hostname=`hostname -f` # Unmount device /etc/hosts and replace it by a shared hosts file @@ -36,14 +32,3 @@ bigtop_yumrepo_uri,http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0. jdk_package_name,java-1.7.0-openjdk-devel.x86_64 components,hadoop,hbase,yarn,mapred-app EOF - -mkdir -p /data/{1,2} - -# Setup rng-tools to improve virtual machine entropy performance. -# The poor entropy performance will cause kerberos provisioning failed. -yum -y install rng-tools -sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd -service rngd start - -echo "Now installing gradle" -cd /bigtop-home && puppet apply --modulepath=./ -e "include bigtop_toolchain::gradle" # alias gradle=/usr/local/gradle/bin/gradle http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/docker-puppet/vagrantconfig.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/vagrantconfig.yaml b/bigtop-deploy/vm/docker-puppet/vagrantconfig.yaml new file mode 100644 index 0000000..e1aac7a --- /dev/null +++ b/bigtop-deploy/vm/docker-puppet/vagrantconfig.yaml @@ -0,0 +1,11 @@ +docker: + memory_size: "2048" + image: "bigtop/ssh:centos-6.4" + +boot2docker: + memory_size: "4196" + number_cpus: "2" + +namenode_ui_port: "50070" +yarn_ui_port: "8088" +hbase_ui_port: "60010" http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/smoke-tests.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/smoke-tests.sh b/bigtop-deploy/vm/smoke-tests.sh deleted file mode 100755 index abc0c09..0000000 --- a/bigtop-deploy/vm/smoke-tests.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -HCFS_USER="hdfs" - -export HADOOP_CONF_DIR=/etc/hadoop/conf/ -export BIGTOP_HOME=/bigtop-home/ -export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/ -export HIVE_HOME=/usr/lib/hive/ -export PIG_HOME=/usr/lib/pig/ -export FLUME_HOME=/usr/lib/flume/ -export HIVE_CONF_DIR=/etc/hive/conf/ -export JAVA_HOME="/usr/lib/jvm/java-openjdk/" -export MAHOUT_HOME="/usr/lib/mahout" -export ITEST="0.7.0" - -su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root' -su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant' -su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root' - -yum install -y pig hive flume mahout -cd /bigtop-home/bigtop-tests/smoke-tests && ./gradlew clean compileGroovy test -Dsmoke.tests=mapreduce,pig --info http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/utils/hbase-test.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/utils/hbase-test.sh b/bigtop-deploy/vm/utils/hbase-test.sh new file mode 100755 index 0000000..89b2b76 --- /dev/null +++ b/bigtop-deploy/vm/utils/hbase-test.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +vagrant ssh bigtop1 -c "hbase shell <<EOF +create 't1','cf1' +put 't1', 'row1', 'cf1:q1', 'value1' +scan 't1' +disable 't1' +drop 't1' +EOF" http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/utils/setup-env.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/utils/setup-env.sh b/bigtop-deploy/vm/utils/setup-env.sh new file mode 100755 index 0000000..d211fc9 --- /dev/null +++ b/bigtop-deploy/vm/utils/setup-env.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Install puppet agent +yum -y install http://yum.puppetlabs.com/puppetlabs-release-el-6.noarch.rpm +yum -y install puppet + +mkdir -p /data/{1,2} + +# Setup rng-tools to improve virtual machine entropy performance. +# The poor entropy performance will cause kerberos provisioning failed. +yum -y install rng-tools +sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd +service rngd start + +echo "Now installing gradle" +cd /bigtop-home && puppet apply --modulepath=./ -e "include bigtop_toolchain::gradle" # alias gradle=/usr/local/gradle/bin/gradle http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/utils/smoke-tests.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/utils/smoke-tests.sh b/bigtop-deploy/vm/utils/smoke-tests.sh new file mode 100755 index 0000000..abc0c09 --- /dev/null +++ b/bigtop-deploy/vm/utils/smoke-tests.sh @@ -0,0 +1,20 @@ +#!/bin/bash +HCFS_USER="hdfs" + +export HADOOP_CONF_DIR=/etc/hadoop/conf/ +export BIGTOP_HOME=/bigtop-home/ +export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/ +export HIVE_HOME=/usr/lib/hive/ +export PIG_HOME=/usr/lib/pig/ +export FLUME_HOME=/usr/lib/flume/ +export HIVE_CONF_DIR=/etc/hive/conf/ +export JAVA_HOME="/usr/lib/jvm/java-openjdk/" +export MAHOUT_HOME="/usr/lib/mahout" +export ITEST="0.7.0" + +su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root' +su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant' +su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root' + +yum install -y pig hive flume mahout +cd /bigtop-home/bigtop-tests/smoke-tests && ./gradlew clean compileGroovy test -Dsmoke.tests=mapreduce,pig --info http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/vagrant-puppet/README.md ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet/README.md b/bigtop-deploy/vm/vagrant-puppet/README.md index 91acadd..001a262 100644 --- a/bigtop-deploy/vm/vagrant-puppet/README.md +++ b/bigtop-deploy/vm/vagrant-puppet/README.md @@ -48,26 +48,26 @@ vagrant plugin install vagrant-cachier vagrant up ``` -4) You can specify number of nodes you'd like to provision by modifying `num_instances` in Vagrantfile +4) You can specify number of nodes you'd like to provision by modifying `num_instances` in vagrantconfig.yaml ``` -num_instances = 5 +num_instances: 5 ``` -##Example: - -5) Run hbase-test.sh to evaluate the deployment. - ##Configure Apache Hadoop ecosystem components -* Choose the ecosystem you want to be deployed by modifying components in provision.sh. +* Choose the ecosystem you want to be deployed by modifying components in vagrantconfig.yaml ``` - components,hadoop,hbase,yarn,mapred-app,... +components: "hadoop,hbase,yarn,mapred-app,..." ``` By default, Apache Hadoop, YARN, and Apache HBase will be installed. See `bigtop-deploy/puppet/config/site.csv.example` for more details. -##Note: +##Example: -For bigtop 0.7.0 code base, you must change the value of the [yarn-site.xml](https://github.com/apache/bigtop/blob/master/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml) yarn.nodemanager.aux.services from "mapreduce_shuffle" to "mapreduce.shuffle" before `vagrant up` +5) Run hbase-test.sh to evaluate the deployment. + +``` +../utils/hbase-test.sh +``` http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/vagrant-puppet/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile index b1ef5b5..d5eca4a 100755 --- a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile +++ b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile @@ -16,26 +16,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +require "yaml" + +_config = YAML.load(File.open(File.join(File.dirname(__FILE__), "vagrantconfig.yaml"), File::RDONLY).read) +CONF = _config + VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - vmbox = "puppetlab-centos-64-nocm" - # Repository # Example for testing a Release candidate. - repo = "http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/"; + repo = CONF['repo'] # repo = "http://bigtop.s3.amazonaws.com/releases/0.7.0/redhat/6/x86_64"; - # Virtual box implementation - vmboxurl = "http://puppet-vagrant-boxes.puppetlabs.com/centos-64-x64-vbox4210-nocm.box"; - # number of instances - num_instances = 1 + num_instances = CONF['num_instances'] # manage /etc/hosts by hostmanager plugin(https://github.com/smdahlen/vagrant-hostmanager) # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier) - if Vagrant.has_plugin?("vagrant-hostmanager") - config.hostmanager.enabled = true - end + config.hostmanager.enabled = true # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier) if Vagrant.has_plugin?("vagrant-cachier") @@ -49,8 +47,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| (1..num_instances).each do |i| config.vm.define "bigtop#{i}" do |bigtop| - bigtop.vm.box = vmbox - bigtop.vm.box_url = vmboxurl + bigtop.vm.box = CONF['box'] bigtop_hostname="bigtop#{i}.vagrant" bigtop_ip="10.10.10.1#{i}" if "#{i}" == "1" @@ -58,7 +55,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end bigtop.vm.provider :virtualbox do |vb| - vb.customize ["modifyvm", :id, "--memory", "10024"] + vb.customize ["modifyvm", :id, "--memory", CONF['memory_size']] + vb.customize ['modifyvm', :id, '--cpus', CONF['number_cpus']] end bigtop.vm.network :private_network, ip: bigtop_ip @@ -68,10 +66,12 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # the current directory has puppet recipes which we need for provisioning. bigtop.vm.synced_folder "../../../", "/bigtop-home" + bigtop.vm.provision "shell", path: "../utils/setup-env.sh" + # shell provisioner to setup basic vm environment bigtop.vm.provision :shell do |shell| - shell.path = "provision.sh" - shell.args = [ bigtop_master , repo ] + shell.path = "provision.sh" + shell.args = [ bigtop_master , repo, CONF["components"]] end bigtop.vm.provision :hostmanager @@ -81,12 +81,12 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| puppet.module_path = "../../puppet/modules/" puppet.manifests_path = "../../puppet/manifests/" puppet.manifest_file = "site.pp" - puppet.options = "--debug --confdir=/bigtop-home/bigtop-deploy/puppet/" + puppet.options = "--debug --confdir=/vagrant" end if i==num_instances then - puts "creating provisioner directive for running tests" - bigtop.vm.provision "shell", path: "../smoke-tests.sh" + puts "creating provisioner directive for running tests" + bigtop.vm.provision "shell", path: "../utils/smoke-tests.sh" else puts "Not creating provisioner directive for tests yet... only on vm #{i} of #{num_instances}" end http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/vagrant-puppet/hbase-test.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet/hbase-test.sh b/bigtop-deploy/vm/vagrant-puppet/hbase-test.sh deleted file mode 100755 index 89b2b76..0000000 --- a/bigtop-deploy/vm/vagrant-puppet/hbase-test.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -vagrant ssh bigtop1 -c "hbase shell <<EOF -create 't1','cf1' -put 't1', 'row1', 'cf1:q1', 'value1' -scan 't1' -disable 't1' -drop 't1' -EOF" http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/vagrant-puppet/provision.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet/provision.sh b/bigtop-deploy/vm/vagrant-puppet/provision.sh index 4ab42d9..4aeee57 100755 --- a/bigtop-deploy/vm/vagrant-puppet/provision.sh +++ b/bigtop-deploy/vm/vagrant-puppet/provision.sh @@ -15,10 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Install puppet agent -yum -y install http://yum.puppetlabs.com/puppetlabs-release-el-6.noarch.rpm -yum -y install puppet - service iptables stop chkconfig iptables off cat /dev/null > /etc/hosts @@ -26,18 +22,11 @@ cat /dev/null > /etc/hosts echo "Bigtop yum repo = $2" # Prepare puppet configuration file -cat > /bigtop-home/bigtop-deploy/puppet/config/site.csv << EOF +mkdir /vagrant/config +cat > /vagrant/config/site.csv << EOF hadoop_head_node,$1 hadoop_storage_dirs,/data/1,/data/2 bigtop_yumrepo_uri,$2 jdk_package_name,java-1.7.0-openjdk-devel.x86_64 -components,hadoop,hbase,yarn,mapred-app +components,$3 EOF - -mkdir -p /data/{1,2} - -# Setup rng-tools to improve virtual machine entropy performance. -# The poor entropy performance will cause kerberos provisioning failed. -yum -y install rng-tools -sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd -service rngd start http://git-wip-us.apache.org/repos/asf/bigtop/blob/801bade7/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml new file mode 100644 index 0000000..d045ce2 --- /dev/null +++ b/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml @@ -0,0 +1,6 @@ +memory_size: 1024 +number_cpus: 1 +box: "puppetlabs/centos-6.5-64-nocm" +repo: "http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/"; +num_instances: 3 +components: "hadoop,hbase,yarn,mapred-app"
