Repository: bigtop Updated Branches: refs/heads/master e209fdbbb -> e722c9ced
BIGTOP-1498. Add RC Testing to Docker based Bigtop provisioner Signed-off-by: [email protected] <jayunit100> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/e722c9ce Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/e722c9ce Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/e722c9ce Branch: refs/heads/master Commit: e722c9ced7ce4f89b43623f1ff04a4c1a7b75369 Parents: e209fdb Author: evans_ye <[email protected]> Authored: Mon Oct 27 23:49:03 2014 +0800 Committer: [email protected] <jayunit100> Committed: Mon Oct 27 23:41:12 2014 -0400 ---------------------------------------------------------------------- bigtop-deploy/vm/docker-puppet/README.md | 18 ++++++++++++----- bigtop-deploy/vm/docker-puppet/Vagrantfile | 6 +++--- bigtop-deploy/vm/docker-puppet/docker-hadoop.sh | 18 +++++++++++++---- bigtop-deploy/vm/docker-puppet/provision.sh | 6 +++++- bigtop-deploy/vm/smoke-tests.sh | 21 ++++++++++++++++++++ bigtop-deploy/vm/vagrant-puppet/Vagrantfile | 2 +- bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh | 21 -------------------- 7 files changed, 57 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/e722c9ce/bigtop-deploy/vm/docker-puppet/README.md ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/README.md b/bigtop-deploy/vm/docker-puppet/README.md index 96b9a1a..70f347e 100644 --- a/bigtop-deploy/vm/docker-puppet/README.md +++ b/bigtop-deploy/vm/docker-puppet/README.md @@ -94,10 +94,16 @@ In case of errors you can attempt running as root, or else, ping the mailing lis ./docker-hadoop.sh --provision ``` -5) Chain your operations with-in one command. +5) Run Bigtop smoke tests ``` -./docker-hadoop.sh --build-image --create 5 --destroy +./docker-hadoop.sh --smoke-tests +``` + +6) Chain your operations with-in one command. + +``` +./docker-hadoop.sh --build-image --create 5 --smoke-tests --destroy ``` Commands will be executed by following order: @@ -106,23 +112,25 @@ Commands will be executed by following order: build-image => create 5 node cluster => destroy the cluster ``` -6) Run hbase-test.sh to evaluate the deployment. +7) Run hbase-test.sh to evaluate the deployment. ``` ../vagrant-puppet/hbase-test.sh ``` -7) See helper message: +8) See helper message: ``` ./docker-hadoop.sh -h usage: docker-hadoop.sh [options] -b, --build-image Build base Docker image for Bigtop Hadoop (must be exectued at least once before creating cluster) - -c NUM_INSTANCES, --create NUM_INSTANCES Create a docker based Bigtop Hadoop cluster + -c NUM_INSTANCES, --create=NUM_INSTANCES Create a docker based Bigtop Hadoop cluster -p, --provision Deploy configuration changes + -s, --smoke-tests Run Bigtop smoke tests -d, --destroy Destroy the cluster -h, --help + ``` ##Configure Apache Hadoop ecosystem components http://git-wip-us.apache.org/repos/asf/bigtop/blob/e722c9ce/bigtop-deploy/vm/docker-puppet/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/Vagrantfile b/bigtop-deploy/vm/docker-puppet/Vagrantfile index ea7153a..fd73144 100755 --- a/bigtop-deploy/vm/docker-puppet/Vagrantfile +++ b/bigtop-deploy/vm/docker-puppet/Vagrantfile @@ -20,8 +20,6 @@ require 'fileutils' # number of instances $num_instances = 3 -# the bigtop puppet recipes -$bigtop_puppet = "../../puppet" # load configuration from config.rb CONFIG = File.join(File.dirname(__FILE__), "config.rb") @@ -31,6 +29,8 @@ end ENV['VAGRANT_DEFAULT_PROVIDER'] ||= 'docker' +Vagrant.require_version ">= 1.6.0" + VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| @@ -62,7 +62,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| bigtop.ssh.username = "root" bigtop.ssh.port = 22 bigtop.vm.hostname = "bigtop#{i}.docker" - bigtop.vm.synced_folder $bigtop_puppet, "/bigtop-puppet" + bigtop.vm.synced_folder "../../../", "/bigtop-home" end end http://git-wip-us.apache.org/repos/asf/bigtop/blob/e722c9ce/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh b/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh index 449b5fb..3650c9c 100755 --- a/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh +++ b/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh @@ -4,8 +4,9 @@ usage() { echo "usage: $PROG [options]" echo " -b, --build-image Build base Docker image for Bigtop Hadoop" echo " (must be exectued at least once before creating cluster)" - echo " -c NUM_INSTANCES, --create=NUM_INSTANCES Create a docker based Bigtop Hadoop cluster" + echo " -c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster" echo " -p, --provision Deploy configuration changes" + echo " -s, --smoke-tests Run Bigtop smoke tests" echo " -d, --destroy Destroy the cluster" echo " -h, --help" exit 1 @@ -46,17 +47,23 @@ provision() { wait } +smoke-tests() { + nodes=(`vagrant status |grep running |awk '{print $1}'`) + echo "/bigtop-home/bigtop-deploy/vm/smoke-tests.sh" |vagrant ssh ${nodes[0]} +} + + destroy() { vagrant destroy -f - rm -f ./hosts + rm -rf ./hosts ./config } bigtop-puppet() { - echo "puppet apply -d --confdir=/bigtop-puppet --modulepath=/bigtop-puppet/modules /bigtop-puppet/manifests/site.pp" |vagrant ssh $1 + echo "puppet apply -d --confdir=/vagrant --modulepath=/bigtop-home/bigtop-deploy/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests/site.pp" |vagrant ssh $1 } PROG=`basename $0` -ARGS=`getopt -o "bc:pdh" -l "build-image,create:,provision,destroy,help" -n $PROG -- "$@"` +ARGS=`getopt -o "bc:psdh" -l "build-image,create:,provision,smoke-tests,destroy,help" -n $PROG -- "$@"` if [ $? -ne 0 ]; then usage @@ -75,6 +82,9 @@ while true; do -p|--provision) provision shift;; + -s|--smoke-tests) + smoke-tests + shift;; -d|--destroy) destroy shift;; http://git-wip-us.apache.org/repos/asf/bigtop/blob/e722c9ce/bigtop-deploy/vm/docker-puppet/provision.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/docker-puppet/provision.sh b/bigtop-deploy/vm/docker-puppet/provision.sh index 929999d..5bd3605 100755 --- a/bigtop-deploy/vm/docker-puppet/provision.sh +++ b/bigtop-deploy/vm/docker-puppet/provision.sh @@ -28,7 +28,8 @@ mv /etc/hosts /etc/hosts.bak ln -s /vagrant/hosts /etc/hosts # Prepare puppet configuration file -cat > /bigtop-puppet/config/site.csv << EOF +mkdir /vagrant/config +cat > /vagrant/config/site.csv << EOF hadoop_head_node,$1 hadoop_storage_dirs,/data/1,/data/2 bigtop_yumrepo_uri,http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/ @@ -43,3 +44,6 @@ mkdir -p /data/{1,2} yum -y install rng-tools sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd service rngd start + +echo "Now installing gradle" +cd /bigtop-home && puppet apply --modulepath=./ -e "include bigtop_toolchain::gradle" # alias gradle=/usr/local/gradle/bin/gradle http://git-wip-us.apache.org/repos/asf/bigtop/blob/e722c9ce/bigtop-deploy/vm/smoke-tests.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/smoke-tests.sh b/bigtop-deploy/vm/smoke-tests.sh new file mode 100755 index 0000000..d02c128 --- /dev/null +++ b/bigtop-deploy/vm/smoke-tests.sh @@ -0,0 +1,21 @@ +#!/bin/bash +HCFS_USER="hdfs" +gradle="/usr/local/gradle/bin/gradle" + +export HADOOP_CONF_DIR=/etc/hadoop/conf/ +export BIGTOP_HOME=/bigtop-home/ +export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/ +export HIVE_HOME=/usr/lib/hive/ +export PIG_HOME=/usr/lib/pig/ +export FLUME_HOME=/usr/lib/flume/ +export HIVE_CONF_DIR=/etc/hive/conf/ +export JAVA_HOME="/usr/lib/jvm/java-1.7.0-openjdk-1.7.0.65.x86_64/" +export MAHOUT_HOME="/usr/lib/mahout" +export ITEST="0.7.0" + +su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root' +su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant' +su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root' + +yum install -y pig hive flume mahout +cd /bigtop-home/bigtop-tests/smoke-tests && $gradle compileGroovy test -Dsmoke.tests=pig --info http://git-wip-us.apache.org/repos/asf/bigtop/blob/e722c9ce/bigtop-deploy/vm/vagrant-puppet/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile index ffe23a9..b1ef5b5 100755 --- a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile +++ b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile @@ -86,7 +86,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| if i==num_instances then puts "creating provisioner directive for running tests" - bigtop.vm.provision "shell", path: "smoke-tests.sh" + bigtop.vm.provision "shell", path: "../smoke-tests.sh" else puts "Not creating provisioner directive for tests yet... only on vm #{i} of #{num_instances}" end http://git-wip-us.apache.org/repos/asf/bigtop/blob/e722c9ce/bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh b/bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh deleted file mode 100755 index d02c128..0000000 --- a/bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -HCFS_USER="hdfs" -gradle="/usr/local/gradle/bin/gradle" - -export HADOOP_CONF_DIR=/etc/hadoop/conf/ -export BIGTOP_HOME=/bigtop-home/ -export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/ -export HIVE_HOME=/usr/lib/hive/ -export PIG_HOME=/usr/lib/pig/ -export FLUME_HOME=/usr/lib/flume/ -export HIVE_CONF_DIR=/etc/hive/conf/ -export JAVA_HOME="/usr/lib/jvm/java-1.7.0-openjdk-1.7.0.65.x86_64/" -export MAHOUT_HOME="/usr/lib/mahout" -export ITEST="0.7.0" - -su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root' -su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant' -su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root' - -yum install -y pig hive flume mahout -cd /bigtop-home/bigtop-tests/smoke-tests && $gradle compileGroovy test -Dsmoke.tests=pig --info
