Repository: bigtop Updated Branches: refs/heads/master 1472a0e11 -> b00e9da8f
BIGTOP-1655. Update VM and Docker provisioner to work with hiera converted puppet recipes Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/b00e9da8 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/b00e9da8 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/b00e9da8 Branch: refs/heads/master Commit: b00e9da8f2b98ffbd0bc7233a1a5452b83a9271b Parents: 1472a0e Author: Evans Ye <[email protected]> Authored: Sat Feb 7 17:18:12 2015 +0000 Committer: Evans Ye <[email protected]> Committed: Sun Feb 8 14:09:20 2015 +0000 ---------------------------------------------------------------------- .../vm/vagrant-puppet-docker/Vagrantfile | 2 +- .../vm/vagrant-puppet-docker/docker-hadoop.sh | 29 ++++++++++++-------- .../vm/vagrant-puppet-docker/provision.sh | 16 ++++++----- .../vm/vagrant-puppet-docker/vagrantconfig.yaml | 2 +- .../vagrantconfig_debian.yaml | 2 +- bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile | 18 ++++++------ .../vm/vagrant-puppet-vm/vagrantconfig.yaml | 2 +- 7 files changed, 40 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile index f800df7..f082142 100755 --- a/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile +++ b/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile @@ -20,7 +20,7 @@ require 'fileutils' # number of instances $num_instances = 3 -#$vagrantyamlconf = "vagrantconfig.yaml" +$vagrantyamlconf = "vagrantconfig.yaml" # load configuration from config.rb CONFIG = File.join(File.dirname(__FILE__), "config.rb") http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh b/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh index 3b44c4f..6459e85 100755 --- a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh +++ b/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh @@ -32,24 +32,29 @@ create() { nodes=(`vagrant status |grep running |grep -v image |awk '{print $1}'`) hadoop_head_node=(`echo "hostname -f" |vagrant ssh ${nodes[0]} |tail -n 1`) repo=$(get-yaml-config repo) - components=$(get-yaml-config components) + components="[`echo $(get-yaml-config components) | sed 's/ /, /g'`]" distro=$(get-yaml-config distro) - echo "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$distro.sh" |vagrant ssh ${nodes[0]} - echo "/vagrant/provision.sh $hadoop_head_node $repo $components" |vagrant ssh ${nodes[0]} - bigtop-puppet ${nodes[0]} - for ((i=1 ; i<${#nodes[*]} ; i++)); do + + # setup environment before running bigtop puppet deployment + for node in ${nodes[*]}; do ( - echo "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$distro.sh" |vagrant ssh ${nodes[$i]} - echo "/vagrant/provision.sh $hadoop_head_node $repo $components" |vagrant ssh ${nodes[$i]} - bigtop-puppet ${nodes[$i]} + echo "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$distro.sh" |vagrant ssh $node + echo "/vagrant/provision.sh $hadoop_head_node $repo \"$components\"" |vagrant ssh $node ) & done wait + + # run bigtop puppet (master node need to be provisioned before slave nodes) + bigtop-puppet ${nodes[0]} + for ((i=1 ; i<${#nodes[*]} ; i++)); do + bigtop-puppet ${nodes[$i]} & + done + wait } provision() { nodes=(`vagrant status |grep running |grep -v image |awk '{print $1}'`) - for node in $nodes; do + for node in ${nodes[*]}; do bigtop-puppet $node & done wait @@ -62,16 +67,16 @@ smoke-tests() { destroy() { - rm -rf ./hosts ./config ./config.rb nodes=(`vagrant status |grep running |grep -v image |awk '{print $1}'`) - for node in $nodes; do + rm -rvf ./hosts ./config.rb + for node in ${nodes[*]}; do vagrant destroy -f $node done wait } bigtop-puppet() { - echo "puppet apply -d --confdir=/vagrant --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules:/usr/share/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests/site.pp" |vagrant ssh $1 + echo "puppet apply -d --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules:/usr/share/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests/site.pp" |vagrant ssh $1 } get-yaml-config() { http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh b/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh index 8d1fa90..b9f1cfb 100755 --- a/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh +++ b/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh @@ -32,11 +32,13 @@ else jdk="java-1.7.0-openjdk-devel.x86_64" fi -mkdir /vagrant/config -cat > /vagrant/config/site.csv << EOF -hadoop_head_node,$1 -hadoop_storage_dirs,/data/1,/data/2 -bigtop_yumrepo_uri,$2 -jdk_package_name,$jdk -components,$3 +mkdir -p /etc/puppet/hieradata +cp /bigtop-home/bigtop-deploy/puppet/hiera.yaml /etc/puppet +cp -r /bigtop-home/bigtop-deploy/puppet/hieradata/bigtop/ /etc/puppet/hieradata/ +cat > /etc/puppet/hieradata/site.yaml << EOF +bigtop::hadoop_head_node: $1 +hadoop::hadoop_storage_dirs: [/data/1, /data/2] +Bigtop::bigtop_yumrepo_uri: $2 +bigtop::jdk_package_name: $jdk +hadoop_cluster_node::cluster_components: $3 EOF http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml index bb540e2..c5b75b4 100644 --- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml +++ b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml @@ -8,7 +8,7 @@ boot2docker: repo: "http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/"; distro: centos -components: "hadoop,yarn" +components: [hadoop, yarn] namenode_ui_port: "50070" yarn_ui_port: "8088" hbase_ui_port: "60010" http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml index bd22455..5a8e772 100644 --- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml +++ b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml @@ -8,7 +8,7 @@ boot2docker: repo: "http://to.do/notyet"; distro: debian -components: "hadoop,yarn" +components: [hadoop, yarn] namenode_ui_port: "50070" yarn_ui_port: "8088" hbase_ui_port: "60010" http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile index 2f49b4c..8b49f09 100755 --- a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile +++ b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile @@ -47,13 +47,15 @@ chkconfig iptables off cat /dev/null > /etc/hosts echo "Bigtop yum repo = #{repo}" # Prepare puppet configuration file -mkdir /vagrant/config -cat > /vagrant/config/site.csv << EOF -hadoop_head_node,#{bigtop_master} -hadoop_storage_dirs,/data/1,/data/2 -bigtop_yumrepo_uri,#{repo} -jdk_package_name,java-1.7.0-openjdk-devel.x86_64 -components,#{components} +mkdir -p /etc/puppet/hieradata +cp /bigtop-home/bigtop-deploy/puppet/hiera.yaml /etc/puppet +cp -r /bigtop-home/bigtop-deploy/puppet/hieradata/bigtop/ /etc/puppet/hieradata/ +cat > /etc/puppet/hieradata/site.yaml << EOF +bigtop::hadoop_head_node: #{bigtop_master} +hadoop::hadoop_storage_dirs: [/data/1, /data/2] +Bigtop::bigtop_yumrepo_uri: #{repo} +bigtop::jdk_package_name: java-1.7.0-openjdk-devel.x86_64 +hadoop_cluster_node::cluster_components: #{components} EOF SCRIPT @@ -108,7 +110,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| puppet.module_path = "../../puppet/modules/" puppet.manifests_path = "../../puppet/manifests/" puppet.manifest_file = "site.pp" - puppet.options = "--debug --confdir=/vagrant" + puppet.options = "--debug" end if run_smoke_tests then http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml index baa4cb4..390a3b1 100644 --- a/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml +++ b/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml @@ -3,7 +3,7 @@ number_cpus: 1 box: "puppetlabs/centos-6.5-64-nocm" repo: "http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/"; num_instances: 1 -components: "hadoop,yarn" +components: [hadoop, yarn] enable_local_yum: false run_smoke_tests: false distro: centos
