BIGTOP-1771. Let the user specify what components to be tested when running smoke-tests by Docker/VM provisioner
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/a5b1ad7d Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/a5b1ad7d Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/a5b1ad7d Branch: refs/heads/master Commit: a5b1ad7d1a692a25c5960d6ac2971dce5feb73e4 Parents: ad784a9 Author: Evans Ye <[email protected]> Authored: Thu Mar 26 03:07:51 2015 +0000 Committer: Evans Ye <[email protected]> Committed: Wed Apr 1 15:59:23 2015 +0000 ---------------------------------------------------------------------- bigtop-deploy/vm/utils/smoke-tests.sh | 8 +++++--- bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh | 3 ++- bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml | 1 + .../vm/vagrant-puppet-docker/vagrantconfig_debian.yaml | 1 + bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile | 8 +++++++- bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml | 3 ++- 6 files changed, 18 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5b1ad7d/bigtop-deploy/vm/utils/smoke-tests.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/utils/smoke-tests.sh b/bigtop-deploy/vm/utils/smoke-tests.sh index 581e5fb..dff51a5 100755 --- a/bigtop-deploy/vm/utils/smoke-tests.sh +++ b/bigtop-deploy/vm/utils/smoke-tests.sh @@ -1,5 +1,6 @@ #!/bin/bash HCFS_USER="hdfs" +SMOKE_TESTS=${1:-mapreduce,pig} export HADOOP_CONF_DIR=/etc/hadoop/conf/ export BIGTOP_HOME=/bigtop-home/ @@ -7,6 +8,7 @@ export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/ export HIVE_HOME=/usr/lib/hive/ export PIG_HOME=/usr/lib/pig/ export FLUME_HOME=/usr/lib/flume/ +export SQOOP_HOME=/usr/lib/sqoop/ export HIVE_CONF_DIR=/etc/hive/conf/ export JAVA_HOME="/usr/lib/jvm/java-openjdk/" export MAHOUT_HOME="/usr/lib/mahout" @@ -17,8 +19,8 @@ su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant' su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root' if [ -f /etc/debian_version ] ; then - apt-get -y install pig hive flume mahout + apt-get -y install pig hive flume mahout sqoop else - yum install -y pig hive flume mahout + yum install -y pig hive flume mahout sqoop fi -cd /bigtop-home/bigtop-tests/smoke-tests && ./gradlew clean compileGroovy test -Dsmoke.tests=mapreduce,pig --info +cd /bigtop-home/bigtop-tests/smoke-tests && ./gradlew clean compileGroovy test -Dsmoke.tests=$SMOKE_TESTS --info http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5b1ad7d/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh b/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh index d6e95bf..63c18fd 100755 --- a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh +++ b/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh @@ -64,7 +64,8 @@ provision() { smoke-tests() { nodes=(`vagrant status |grep running |grep -v image |awk '{print $1}'`) - echo "/bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh" |vagrant ssh ${nodes[0]} + smoke_test_components="`echo $(get-yaml-config smoke_test_components) | sed 's/ /,/g'`" + echo "/bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh \"$smoke_test_components\"" |vagrant ssh ${nodes[0]} } http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5b1ad7d/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml index fa55671..5eeeb45 100644 --- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml +++ b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml @@ -13,4 +13,5 @@ namenode_ui_port: "50070" yarn_ui_port: "8088" hbase_ui_port: "60010" enable_local_repo: false +smoke_test_components: [mapreduce, pig] jdk: "java-1.7.0-openjdk-devel.x86_64" http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5b1ad7d/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml index 008c15c..012fd36 100644 --- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml +++ b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml @@ -13,4 +13,5 @@ namenode_ui_port: "50070" yarn_ui_port: "8088" hbase_ui_port: "60010" enable_local_repo: false +smoke_test_components: [mapreduce, pig] jdk: "openjdk-7-jdk" http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5b1ad7d/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile index 9cf0476..5384b8c 100755 --- a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile +++ b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile @@ -38,6 +38,9 @@ components = CONF['components'] # Whether to run smoke tests run_smoke_tests = CONF['run_smoke_tests'] +# Smoke test Components to run +smoke_test_components = CONF['smoke_test_components'].join(',') + # This is a update to allow dev packages # Force success - not worried if this step fails, since we generally only use it for development. enable_local_repo = CONF['enable_local_repo'] @@ -120,7 +123,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| if run_smoke_tests then if i==num_instances then puts "creating provisioner directive for running tests" - bigtop.vm.provision "shell", path: "../utils/smoke-tests.sh" + bigtop.vm.provision :shell do |shell| + shell.path = "../utils/smoke-tests.sh" + shell.args = ["#{smoke_test_components}"] + end else puts "Not creating provisioner directive for tests yet... only on vm #{i} of #{num_instances}" end http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5b1ad7d/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml index 33ef53b..0b3b84d 100644 --- a/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml +++ b/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml @@ -3,8 +3,9 @@ number_cpus: 1 box: "puppetlabs/centos-6.6-64-nocm" repo: "http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/"; num_instances: 1 +distro: centos components: [hadoop, yarn] enable_local_repo: false run_smoke_tests: false -distro: centos +smoke_test_components: [mapredcue, pig] jdk: "java-1.7.0-openjdk-devel.x86_64"
