Repository: bigtop Updated Branches: refs/heads/master 83d070ce0 -> 4ee8f075f
BIGTOP-2647. Clean up code under bigtop-deploy/vm after everything moved to provisioner directory Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/4ee8f075 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/4ee8f075 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/4ee8f075 Branch: refs/heads/master Commit: 4ee8f075f99964d5baf21751f977d54cfab6fb0e Parents: 83d070c Author: Evans Ye <[email protected]> Authored: Sat Jan 21 13:55:57 2017 +0800 Committer: Evans Ye <[email protected]> Committed: Sun Jan 22 22:48:56 2017 +0800 ---------------------------------------------------------------------- bigtop-deploy/vm/utils/hbase-test.sh | 24 --- bigtop-deploy/vm/utils/setup-env-centos.sh | 42 ----- bigtop-deploy/vm/utils/setup-env-debian.sh | 35 ---- bigtop-deploy/vm/utils/smoke-tests.sh | 64 -------- .../vm/vagrant-puppet-docker/.gitignore | 3 - .../vm/vagrant-puppet-docker/README.md | 160 ------------------ .../vm/vagrant-puppet-docker/Vagrantfile | 64 -------- .../boot2docker/Vagrantfile | 38 ----- .../vm/vagrant-puppet-docker/docker-hadoop.sh | 137 ---------------- .../vm/vagrant-puppet-docker/provision.sh | 37 ----- .../vm/vagrant-puppet-docker/vagrantconfig.yaml | 32 ---- .../vagrantconfig_centos-7.yaml | 32 ---- .../vagrantconfig_debian.yaml | 32 ---- .../vagrantconfig_ubuntu.yaml | 32 ---- .../vagrantconfig_ubuntu_ppc64le.yaml | 32 ---- .../vm/vagrant-puppet-openstack/NOTICE | 22 --- .../vm/vagrant-puppet-openstack/README.md | 146 ----------------- .../vm/vagrant-puppet-openstack/Vagrantfile | 153 ------------------ .../vagrant-puppet-openstack/para-provision.sh | 114 ------------- .../vagrant-puppet-openstack/vagrantconfig.yaml | 28 ---- bigtop-deploy/vm/vagrant-puppet-vm/.gitignore | 1 - bigtop-deploy/vm/vagrant-puppet-vm/README.md | 107 ------------ bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile | 161 ------------------- bigtop-deploy/vm/vagrant-puppet-vm/config_hosts | 46 ------ bigtop-deploy/vm/vagrant-puppet-vm/gen_hosts.sh | 44 ----- .../vm/vagrant-puppet-vm/vagrantconfig.yaml | 26 --- 26 files changed, 1612 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/utils/hbase-test.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/utils/hbase-test.sh b/bigtop-deploy/vm/utils/hbase-test.sh deleted file mode 100755 index 89b2b76..0000000 --- a/bigtop-deploy/vm/utils/hbase-test.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -vagrant ssh bigtop1 -c "hbase shell <<EOF -create 't1','cf1' -put 't1', 'row1', 'cf1:q1', 'value1' -scan 't1' -disable 't1' -drop 't1' -EOF" http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/utils/setup-env-centos.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/utils/setup-env-centos.sh b/bigtop-deploy/vm/utils/setup-env-centos.sh deleted file mode 100755 index da45c9f..0000000 --- a/bigtop-deploy/vm/utils/setup-env-centos.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -enable_local_repo=${1:-false} - -bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh - -# Setup rng-tools to improve virtual machine entropy performance. -# The poor entropy performance will cause kerberos provisioning failed. -yum -y install rng-tools -if [ -x /usr/bin/systemctl ] ; then - sed -i 's@ExecStart=/sbin/rngd -f@ExecStart=/sbin/rngd -f -r /dev/urandom@' /usr/lib/systemd/system/rngd.service - systemctl daemon-reload - systemctl start rngd -else - sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd - service rngd start -fi - -if [ $enable_local_repo == "true" ]; then - echo "Enabling local yum." - yum -y install yum-utils - sudo echo "gpgcheck=0" >> /etc/yum.conf - sudo yum-config-manager --add-repo file:///bigtop-home/output -else - echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..." -fi - http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/utils/setup-env-debian.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/utils/setup-env-debian.sh b/bigtop-deploy/vm/utils/setup-env-debian.sh deleted file mode 100755 index ac92615..0000000 --- a/bigtop-deploy/vm/utils/setup-env-debian.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -enable_local_repo=${1:-false} - -bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh - -# Setup rng-tools to improve virtual machine entropy performance. -# The poor entropy performance will cause kerberos provisioning failed. -apt-get -y install rng-tools -sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools -service rng-tools start - -if [ $enable_local_repo == "true" ]; then - echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list - apt-get update -else - apt-get install -y apt-transport-https - echo "local apt = $enable_local_repo ; NOT Enabling local apt. Packages will be pulled from remote..." -fi - http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/utils/smoke-tests.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/utils/smoke-tests.sh b/bigtop-deploy/vm/utils/smoke-tests.sh deleted file mode 100755 index 93f795e..0000000 --- a/bigtop-deploy/vm/utils/smoke-tests.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HCFS_USER="hdfs" -SMOKE_TESTS=${1:-mapreduce,pig} - -# Autodetect JAVA_HOME -if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then - . /usr/lib/bigtop-utils/bigtop-detect-javahome -else - >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n" - exit 2 -fi - -echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n" - -export HADOOP_CONF_DIR=/etc/hadoop/conf/ -export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/ -export HIVE_HOME=/usr/lib/hive/ -export PIG_HOME=/usr/lib/pig/ -export FLUME_HOME=/usr/lib/flume/ -export SQOOP_HOME=/usr/lib/sqoop/ -export HIVE_CONF_DIR=/etc/hive/conf/ -export MAHOUT_HOME="/usr/lib/mahout" - -prep() { - HADOOP_COMMAND=$1 - su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -mkdir /user/vagrant /user/root" - su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/vagrant" - su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/root" -} - -prep hadoop -if [[ $SMOKE_TESTS == *"qfs"* ]]; then - prep hadoop-qfs -fi - -if [ -f /etc/debian_version ] ; then - apt-get -y install pig hive flume mahout sqoop -else - yum install -y pig hive flume mahout sqoop -fi -ALL_SMOKE_TASKS="" -for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do - ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test" -done -cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info -# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same -# permissions as used for smoke-tests execution -rm -rf buildSrc/build/test-results/binary - http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-docker/.gitignore ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/.gitignore b/bigtop-deploy/vm/vagrant-puppet-docker/.gitignore deleted file mode 100644 index 2cd389d..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-docker/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -config.rb -config/ -hosts http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-docker/README.md ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/README.md b/bigtop-deploy/vm/vagrant-puppet-docker/README.md deleted file mode 100644 index fb9121c..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-docker/README.md +++ /dev/null @@ -1,160 +0,0 @@ - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - ------------------------------------------------------------------------------------------------------------------------------------------------------- - -#BigTop Docker provisioner - -## Overview - -The Vagrantfile definition and wrapper script that creates Bigtop virtual Hadoop cluster on top of Docker containers for you, by pulling from existing publishing bigtop repositories. -This cluster can be used: - -- to test bigtop smoke tests -- to test bigtop puppet recipes - -These containers start sshd daemons, which vagrant uses to provision and install the hadoop cluster. - -This has been verified on docker client 1.2.0, with api version 1.15, and vagrant 1.6.5 on Fedora 20 as well as Centos 6. - -## Prerequisites - -### OS X and Windows - -* Install [VirtualBox](https://www.virtualbox.org/wiki/Downloads) - -* Install [Vagrant](http://www.vagrantup.com/downloads.html). Need version 1.6.5 or higher. - -### Linux - -* [Kernel Requirements](http://docker.readthedocs.org/en/v0.5.3/installation/kernel/) - -* Install [Docker](https://docs.docker.com/installation/) - -* Install [Vagrant](http://www.vagrantup.com/downloads.html) - -## Getting Started - -* Create a 3 node Bigtop Hadoop cluster from scratch - -NOTE : SELinux can PREVENT you from ssh'ing into your docker container. -As a brute force way to disable it - remove it from vi /etc/sysconfig/docker arguments -(fedora and centos may by default launch docker daemon with the --selinux-enabled option)! -In the future, lets update this README with the RIGHT way to allow selinux without breaking -ssh into a docker container! - -``` -service docker restart -docker pull bigtop/deploy:centos-6 -``` -Now, you can start your cluster: - -In case of errors you can attempt running as root, or else, ping the mailing list. - -## USAGE - -1) Create a Bigtop Hadoop cluster by given # of node. (will place a file called config.rb) - -``` -./docker-hadoop.sh --create 3 -``` - -2) Destroy the cluster. - -``` -./docker-hadoop.sh --destroy -``` - -3) Update your cluster after doing configuration changes. (re-run puppet apply) - -``` -./docker-hadoop.sh --provision -``` - -4) Run Bigtop smoke tests - -``` -./docker-hadoop.sh --smoke-tests -``` - -5) Chain your operations with-in one command. - -``` -./docker-hadoop.sh --create 5 --smoke-tests --destroy -``` - -Commands will be executed by following order: - -``` -create 5 node cluster => run smoke tests => destroy the cluster -``` - -6) See helper message: - -``` -./docker-hadoop.sh -h -usage: docker-hadoop.sh [options] - -c NUM_INSTANCES, --create NUM_INSTANCES Create a docker based Bigtop Hadoop cluster - -p, --provision Deploy configuration changes - -s, --smoke-tests Run Bigtop smoke tests - -d, --destroy Destroy the cluster - -h, --help - -``` - -##Configurations - -* There are several parameters can be configured in the vagrantconfig.yaml: - -1) Modify memory limit for Docker containers - -``` -docker: - memory_size: "2048" - -``` - -2) If you're running Docker provisioner on OS X or Windows, you can customize the boot2docker VM settings - -``` -boot2docker: - memory_size: "4096" - number_cpus: "2" -``` - -3) Use different host ports mapping for web UIs - -``` -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" - -``` -Note: If running on OS X or Windows, the boot2docker VM should be reloaded after ports changed - - -##Configure Apache Hadoop ecosystem components -* Choose the ecosystem you want to be deployed by modifying components in vagrantconfig.yaml - -``` -components: "hadoop,hbase,yarn,..." -``` - -By default, Apache Hadoop, YARN, and Apache HBase will be installed. -See `bigtop-deploy/puppet/config/site.csv.example` for more details. - -##Notes - -* Users currently using vagrant 1.6+ is strongly recommanded to upgrade to 1.6.4+, otherwise you will encounter the [issue](https://github.com/mitchellh/vagrant/issues/3769) when installing plugins http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile deleted file mode 100755 index 4927501..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile +++ /dev/null @@ -1,64 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require 'fileutils' - -# number of instances -$num_instances = 3 -$vagrantyamlconf = "vagrantconfig.yaml" - -# load configuration from config.rb -CONFIG = File.join(File.dirname(__FILE__), "config.rb") -if File.exist?(CONFIG) - require CONFIG -end - -require "yaml" - -_config = YAML.load(File.open(File.join(File.dirname(__FILE__), $vagrantyamlconf), File::RDONLY).read) -CONF = _config - -ENV['VAGRANT_DEFAULT_PROVIDER'] ||= 'docker' - -Vagrant.require_version ">= 1.6.0" - -VAGRANTFILE_API_VERSION = "2" -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - - # nodes definition - (1..$num_instances).each do |i| - config.vm.define "bigtop#{i}" do |bigtop| - # docker container settings - bigtop.vm.provider "docker" do |d| - d.image = CONF["docker"]["image"] - d.create_args = ["--privileged=true", "-m", CONF["docker"]['memory_size'] + "m"] - if "#{i}" == "1" - d.ports = [CONF["namenode_ui_port"] + ":50070", CONF["yarn_ui_port"] + ":8088", CONF["hbase_ui_port"] + ":60010"] - end - d.vagrant_machine = "boot2docker" - d.vagrant_vagrantfile = "./boot2docker/Vagrantfile" - d.has_ssh = false # We have decided to pipe commands to docker instead of ssh, due to ssh hang issue in vagrant. - end - bigtop.ssh.username = "root" - bigtop.ssh.port = 22 - bigtop.vm.hostname = "bigtop#{i}.docker" - bigtop.vm.synced_folder "../../../", "/bigtop-home" - end - end - -end http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-docker/boot2docker/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/boot2docker/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-docker/boot2docker/Vagrantfile deleted file mode 100644 index 15fe682..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-docker/boot2docker/Vagrantfile +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -*- mode: ruby -*- -# vi: set ft=ruby : - -require "yaml" - -_config = YAML.load(File.open(File.join(File.dirname(__FILE__), "../vagrantconfig.yaml"), File::RDONLY).read) -CONF = _config - -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - - config.vm.define "boot2docker" do |node| - node.vm.box = "yungsang/boot2docker" - node.vm.provider "virtualbox" do |vb| - vb.customize ["modifyvm", :id, "--memory", CONF["boot2docker"]["memory_size"]] - vb.customize ['modifyvm', :id, '--cpus', CONF["boot2docker"]['number_cpus']] - end - node.vm.network "forwarded_port", host: CONF["namenode_ui_port"], guest: 50070 - node.vm.network "forwarded_port", host: CONF["yarn_ui_port"], guest: 8088 - node.vm.network "forwarded_port", host: CONF["hbase_ui_port"], guest: 60010 - end - -end http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh b/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh deleted file mode 100755 index 8c3628b..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -usage() { - echo "usage: $PROG [-C file ] args" - echo " -C file Use alternate file for vagrantconfig.yaml" - echo " commands:" - echo " -c NUM_INSTANCES, --create NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster" - echo " -p, --provision Deploy configuration changes" - echo " -s, --smoke-tests Run Bigtop smoke tests" - echo " -d, --destroy Destroy the cluster" - echo " -h, --help" - exit 1 -} - -create() { - echo "\$num_instances = $1" > config.rb - echo "\$vagrantyamlconf = \"$vagrantyamlconf\"" >> config.rb - vagrant up --no-parallel - if [ $? -ne 0 ]; then - echo "Docker container(s) startup failed!"; - exit 1; - fi - nodes=(`vagrant status |grep bigtop |awk '{print $1}'`) - hadoop_head_node=(`echo "hostname -f" |vagrant ssh ${nodes[0]} |tail -n 1`) - repo=$(get-yaml-config repo) - components="[`echo $(get-yaml-config components) | sed 's/ /, /g'`]" - jdk=$(get-yaml-config jdk) - distro=$(get-yaml-config distro) - enable_local_repo=$(get-yaml-config enable_local_repo) - - # setup environment before running bigtop puppet deployment - for node in ${nodes[*]}; do - ( - echo "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$distro.sh $enable_local_repo" |vagrant ssh $node - echo "/vagrant/provision.sh $hadoop_head_node $repo \"$components\" $jdk" |vagrant ssh $node - ) & - done - wait - - # run bigtop puppet (master node need to be provisioned before slave nodes) - bigtop-puppet ${nodes[0]} - for ((i=1 ; i<${#nodes[*]} ; i++)); do - bigtop-puppet ${nodes[$i]} & - done - wait -} - -provision() { - nodes=(`vagrant status |grep bigtop |awk '{print $1}'`) - for node in ${nodes[*]}; do - bigtop-puppet $node & - done - wait -} - -smoke-tests() { - nodes=(`vagrant status |grep bigtop |awk '{print $1}'`) - smoke_test_components="`echo $(get-yaml-config smoke_test_components) | sed 's/ /,/g'`" - echo "/bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh \"$smoke_test_components\"" |vagrant ssh ${nodes[0]} -} - - -destroy() { - vagrant destroy -f - rm -rvf ./hosts ./config.rb -} - -bigtop-puppet() { - echo "puppet apply --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests/site.pp" |vagrant ssh $1 -} - -get-yaml-config() { - RUBY_EXE=ruby - which ruby > /dev/null 2>&1 - if [ $? -ne 0 ]; then - # use vagrant embedded ruby on Windows - RUBY_EXE=$(dirname $(which vagrant))/../embedded/bin/ruby - fi - RUBY_SCRIPT="data = YAML::load(STDIN.read); puts data['$1'];" - cat ${vagrantyamlconf} | $RUBY_EXE -ryaml -e "$RUBY_SCRIPT" | tr -d '\r' -} - -PROG=`basename $0` - -if [ $# -eq 0 ]; then - usage -fi - -vagrantyamlconf="vagrantconfig.yaml" -while [ $# -gt 0 ]; do - case "$1" in - -c|--create) - if [ $# -lt 2 ]; then - echo "Create requires a number" 1>&2 - usage - fi - create $2 - shift 2;; - -C|--conf) - if [ $# -lt 2 ]; then - echo "Alternative config file for vagrantconfig.yaml" 1>&2 - usage - fi - vagrantyamlconf=$2 - shift 2;; - -p|--provision) - provision - shift;; - -s|--smoke-tests) - smoke-tests - shift;; - -d|--destroy) - destroy - shift;; - -h|--help) - usage - shift;; - *) - echo "Unknown argument: '$1'" 1>&2 - usage;; - esac -done http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh b/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh deleted file mode 100755 index f5f8f3e..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -sysctl kernel.hostname=`hostname -f` - -# Unmount device /etc/hosts and replace it by a shared hosts file -echo -e "`hostname -i`\t`hostname -f`" >> /vagrant/hosts -umount /etc/hosts -mv /etc/hosts /etc/hosts.bak -ln -s /vagrant/hosts /etc/hosts - -# Prepare puppet configuration file -mkdir -p /etc/puppet/hieradata -cp /bigtop-home/bigtop-deploy/puppet/hiera.yaml /etc/puppet -cp -r /bigtop-home/bigtop-deploy/puppet/hieradata/bigtop/ /etc/puppet/hieradata/ -cat > /etc/puppet/hieradata/site.yaml << EOF -bigtop::hadoop_head_node: $1 -hadoop::hadoop_storage_dirs: [/data/1, /data/2] -bigtop::bigtop_repo_uri: $2 -hadoop_cluster_node::cluster_components: $3 -bigtop::jdk_package_name: $4 -hadoop::common_hdfs::testonly_hdfs_sshkeys: "yes" -EOF http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml deleted file mode 100644 index 52a8bf4..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_size: "4096" - image: "bigtop/deploy:centos-6" - -boot2docker: - memory_size: "4096" - number_cpus: "1" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/6/x86_64"; -distro: centos -components: [hadoop, yarn] -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" -enable_local_repo: false -smoke_test_components: [mapreduce, pig] -jdk: "java-1.7.0-openjdk-devel.x86_64" http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_centos-7.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_centos-7.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_centos-7.yaml deleted file mode 100644 index 92d7468..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_centos-7.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_size: "4096" - image: "bigtop/deploy:centos-7" - -boot2docker: - memory_size: "4096" - number_cpus: "1" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64"; -distro: centos -components: [hadoop, yarn] -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" -enable_local_repo: false -smoke_test_components: [mapreduce, pig] -jdk: "java-1.7.0-openjdk-devel.x86_64" http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml deleted file mode 100644 index fd4a395..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_size: "4096" - image: "bigtop/deploy:debian-8" - -boot2docker: - memory_size: "4096" - number_cpus: "1" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/debian/8/x86_64"; -distro: debian -components: [hadoop, yarn] -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" -enable_local_repo: false -smoke_test_components: [mapreduce, pig] -jdk: "openjdk-7-jdk" http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_ubuntu.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_ubuntu.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_ubuntu.yaml deleted file mode 100644 index b424b08..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_ubuntu.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_size: "4096" - image: "bigtop/deploy:ubuntu-14.04" - -boot2docker: - memory_size: "4096" - number_cpus: "1" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/ubuntu/trusty/x86_64"; -distro: debian -components: [hadoop, yarn] -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" -enable_local_repo: false -smoke_test_components: [mapreduce, pig] -jdk: "openjdk-7-jdk" http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_ubuntu_ppc64le.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_ubuntu_ppc64le.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_ubuntu_ppc64le.yaml deleted file mode 100644 index d34ba91..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_ubuntu_ppc64le.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_size: "4096" - image: "bigtop/deploy:ubuntu-15.04-ppc64le" - -boot2docker: - memory_size: "4096" - number_cpus: "1" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/ubuntu/vivid/ppc64el"; -distro: debian -components: [hadoop, yarn] -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" -enable_local_repo: false -smoke_test_components: [mapreduce, pig] -jdk: "openjdk-7-jdk" http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-openstack/NOTICE ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-openstack/NOTICE b/bigtop-deploy/vm/vagrant-puppet-openstack/NOTICE deleted file mode 100644 index 53b0f8e..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-openstack/NOTICE +++ /dev/null @@ -1,22 +0,0 @@ -# para-vagrant.sh - -minggLu's `deploy` script is derived from the `para-vagrant.sh` script by Joe Miller, available at -https://github.com/joemiller/sensu-tests/blob/master/para-vagrant.sh. - ---- -Author:: Joe Miller (<[email protected]>) -Copyright:: Copyright (c) 2012 Joe Miller -License:: Apache License, Version 2.0 - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---- http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-openstack/README.md ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-openstack/README.md b/bigtop-deploy/vm/vagrant-puppet-openstack/README.md deleted file mode 100644 index cf5828b..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-openstack/README.md +++ /dev/null @@ -1,146 +0,0 @@ - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - ----------------------------------------------------------------------------- - -# BigTop OpenStack VM Provisioner - -## Overview - -This vagrant recipe is based on the vagrant recipe from `vagrant-puppet-vm` with added feature of vagrant-openstack-provider plugin. The plugin allows us to deploy a Hadoop cluster on an actual virtual environment as if we are deploying on local vagrant vms. It will spin up and provision the vm(s) for us. - -The Vagrantfile creates a BigTop virtual Hadoop cluster on OpenStack by using BigTop puppet recipes and pulling from existing bigtop repositories - -When the configuration is correctly set up in vagrantconfig.yaml, we should be able to deploy a cluster with on single command `vagrant up` - -This can be use: - -* to deploy BigTop Hadoop cluster(s) on an OpenStack cloud environment -* to run BigTop smoke tests on the cluster - -## Usage - -0) Set up environment - -Install vagrant from [official website](www.vagrantup.com) - -If you want to provision machines in parallel, install gnu-parallel - -``` -# for centos -yum install parallel -# for mac -brew install parallel -# for debian/ubuntu -apt-get install parallel -``` - -1) Install [vagrant-hostmanager plugin](https://github.com/smdahlen/vagrant-hostmanager) to better manage `/etc/hosts` - -``` -vagrant plugin install vagrant-hostmanager -``` - -2) Install [vagrant-openstack-provider](https://github.com/ggiamarchi/vagrant-openstack-provider) - -``` -vagrant plugin install vagrant-openstack-provider -``` - -3) Set up configuration - -For now this is partically handled by openstack rc file and partically handled by vagrantconfig.yaml file - -Download rc file from Openstack Horizon dashbord Access & Security and run -``` -source projectname-openrc.sh -``` -You will also need to specify flavor, image_id, keypair_name, and FQDN (fully qualified domain name of your openstack environment) in vagrantconfig.yaml to successfully spin up a vm - -``` -flavor: "name of your choice of flavor" # e.g. m1.small -image_id: "UUID of your choice of image" # e.g. 8fddf8aa-1809-414d-b478-f93b8415f5f4 -keypair_name: "your key pair name on openstack" # e.g. cloud-key -FQDN: "the fully qualified domain name of the environment" -key_path: "location of your private key" #e.g. ~/.ssh/cloud-key.pem -``` - -There are other options in vagrantconfig.yaml that you can specify such as set number of vms in the cluster, and automatically run smoke tests - -The `run_in_parallel` options should be set to true if want to provision machins in parallel. see below for how to. - -``` -num_instance: 1 -run_smoke_tests: true -run_in_parallel: false -``` - -You can also determine what components are being installed and tested - -``` -components: [hadoop, yarn] -smoke_test_components: [mapredcue, pig] -``` - -## GO - -For deployment in sequence - -1. set `run_in_parallel` option in `vagrantconfig.yaml` to false -2. run -``` -vagrant up --provider=openstack -``` - -For parallel provisioning: - -1. set `run_in_parallel` option in `vagrantconfig.yaml` to true -2. run -``` -./para-provision.sh -``` - -#### Parallel provisioning - -**This script is based on Joe Miller's para-vagrant.sh script please see NOTICE for more information** - -Script reads parameter `num_instance`, `run_smoke_tests` and `smoke_test_components` from `vagrantconfig.yaml` to determine how many machines to spin up and whether or not to run smoke tests and what components will be tested - -This script will spin up vms on openstack sequentially first, and then do the provisioning in parallel. Each guest machine will have it's own log file. And will generate a log file for smoke tests if `run_smoke_tests` set to true - -There are some sketchy places in the code...(cuz I suck in bash) such as: -* still have unprintable ^M in log file. I haven't figure out how to get rid of them without destroying the format yet -* use of `sed`: so OS X hates me, and it won't let me use `sed -r` so again I create another temporary file for the smoke tests log - -**NOTE**: -* the name of the vm in `Vagrantfile` **has to** match the name of the vim in `para-provision.sh`, you can change them in line 66 - -``` -for ((i=1; i<=$NUM_INSTANCE; i++)); do - cat <<EOF -name_of_your_vm$i -EOF -``` - -* In `Vagrantfile` the run smoke test part is currently commented out, in order to run smoke test after **all** the machines are done provisioning. The smoke tests are currently being handled by `para-provision.sh`. - -#### TODO - -* test installing all available components - * spark2 is not working, looks like a puppet problem -* test all the provided smoke tests - * mahout smoke tests didn't run all the way throught -* enable_local_repo -* modify the code to make it more generic, I only tried this on Centos 6 http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-openstack/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-openstack/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-openstack/Vagrantfile deleted file mode 100644 index 13e905a..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-openstack/Vagrantfile +++ /dev/null @@ -1,153 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require "yaml" - -_config = YAML.load(File.open(File.join(File.dirname(__FILE__), "vagrantconfig.yaml"), File::RDONLY).read) -CONF = _config - -# Override vagrant configurations using environment variables -keys = CONF.keys -keys.each do |k| - if ENV[k.upcase] != nil then - puts "Overide from environment variable: " + k.upcase + " = " + ENV[k.upcase] - if /^\d+/.match(ENV[k.upcase]) - CONF[k] = Integer(ENV[k.upcase]) - else - CONF[k] = ENV[k.upcase] - end - end -end - -# Repository -# Example for testing a Release candidate. -repo = CONF['repo'] -# repo = "http://bigtop.s3.amazonaws.com/releases/0.7.0/redhat/6/x86_64"; - -# Which Linux Distribution to use. Right now only centos is tested -distro = CONF['distro'] - -# number of instances -num_instances = CONF['num_instances'] - -# hadoop ecosystem components -components = CONF['components'] - -# Whether to run smoke tests -run_smoke_tests = CONF['run_smoke_tests'] - -# Smoke test Components to run -smoke_test_components = CONF['smoke_test_components'].join(',') - -# This is a update to allow dev packages -# Force success - not worried if this step fails, since we generally only use it for development. -enable_local_repo = CONF['enable_local_repo'] -puts "vagrant conf local repo enabled: #{enable_local_repo}" - -# JDK package name -jdk = CONF['jdk'] - -# instance definition -flavor = CONF['flavor'] -image_id = CONF['image_id'] -keypair_name = CONF['keypair_name'] -FQDN = CONF['FQDN'] -key_path = CONF['key_path'] -run_in_parallel = CONF['run_in_parallel'] -require 'vagrant-openstack-provider' - -# master node hostname -# be careful about the fqdn stuff, because its throught actual network -# in this case it's csail.mit.edu -hadoop_master = "hadoop-bigtop1.#{FQDN}" - -$script = <<SCRIPT -service iptables stop -chkconfig iptables off -# Prepare puppet configuration file -mkdir -p /etc/puppet/hieradata -cp /bigtop-home/bigtop-deploy/puppet/hiera.yaml /etc/puppet -cp -r /bigtop-home/bigtop-deploy/puppet/hieradata/bigtop/ /etc/puppet/hieradata/ -cat > /etc/puppet/hieradata/site.yaml << EOF -bigtop::hadoop_head_node: #{hadoop_master} -hadoop::hadoop_storage_dirs: [/data/1, /data/2] -bigtop::bigtop_repo_uri: #{repo} -hadoop_cluster_node::cluster_components: #{components} -bigtop::jdk_package_name: #{jdk} -hadoop::common_hdfs::testonly_hdfs_sshkeys: "yes" -EOF -SCRIPT - -Vagrant.configure(2) do |config| - # enable hostmanager to manage /etc/hosts - config.hostmanager.enabled = true - - # provision (multiple) node(s) - (1..num_instances).each do |i| - config.vm.define "hadoop-bigtop#{i}" do |bigtop| - bigtop.ssh.pty = true - bigtop.ssh.username = 'centos' - bigtop.ssh.private_key_path = key_path - bigtop.vm.provider :openstack do |os| - os.openstack_auth_url = 'https://nimbus.csail.mit.edu:5001/v2.0/tokens' - os.username = ENV['OS_USERNAME'] - os.password = ENV['OS_PASSWORD'] - os.tenant_name = ENV['OS_TENANT_NAME'] - os.flavor = flavor - os.server_name = "hadoop-bigtop#{i}" - os.image = image_id - os.endpoint_type = 'publicURL' - os.keypair_name = keypair_name - os.sync_method = 'rsync' - end - - bigtop.vm.hostname = "hadoop-bigtop#{i}.#{FQDN}" - bigtop.hostmanager.aliases = "hadoop-bigtop#{i}" - - # sync folder from local to vm using rsync - bigtop.vm.synced_folder "../../../", "/bigtop-home" - - # set up environment and hiera and manage hosts - bigtop.vm.provision :shell do |shell| - shell.path = "../utils/setup-env-" + distro + ".sh" - end - bigtop.vm.provision "shell", inline: $script - - # run puppet to deploy hadoop - bigtop.vm.provision :puppet do |puppet| - puppet.module_path = "../../puppet/modules/" - puppet.manifests_path = "../../puppet/manifests/" - puppet.manifest_file = "site.pp" - puppet.options = '--debug' - end - - if run_smoke_tests && !run_in_parallel then - if i==num_instances then - puts "creating provisioner directive for running tests" - bigtop.vm.provision :shell do |shell| - shell.path = "../utils/smoke-tests.sh" - shell.args = ["#{smoke_test_components}"] - end - else - puts "Not creating provisioner directive for tests yet... only on vm #{i} of #{num_instances}" - end - end - end - end - -end http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-openstack/para-provision.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-openstack/para-provision.sh b/bigtop-deploy/vm/vagrant-puppet-openstack/para-provision.sh deleted file mode 100755 index 96f4b1a..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-openstack/para-provision.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/sh -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# concurrency is hard, let's have a beer - -# This script is based onthe `para-vagrant.sh` script by Joe Miller, available at https://github.com/joemiller/sensu-tests/blob/master/para-vagrant.sh. -# see NOTICE file - -# any valid parallel argument will work here, such as -P x. -MAX_PROCS="-j 10" - -# Read parameter from vagrantconfig.yaml file -NUM_INSTANCE=$(grep num_instance vagrantconfig.yaml | awk -F: '/:/{gsub(/ /, "", $2); print $2}') -SMOKE_TEST_COMPONENTS=$(grep smoke_test_components vagrantconfig.yaml | awk -F[ '/,/{gsub(/ /, "", $2); print $2}' | awk -F] '{print $1}') -RUN_SMOKE_TESTS=$(grep run_smoke_tests vagrantconfig.yaml | awk -F: '/:/{gsub(/ /, "", $2); print $2}') - -parallel_provision() { - while read box; do - echo $box - done | parallel $MAX_PROCS -I"NODE" -q \ - sh -c 'LOGFILE="logs/NODE.out.txt" ; \ - printf "[NODE] Provisioning. Log: $LOGFILE, Result: " ; \ - vagrant provision NODE > $LOGFILE 2>&1 ; \ - echo "vagrant provision NODE > $LOGFILE 2>&1" ; \ - RETVAL=$? ; \ - if [ $RETVAL -gt 0 ]; then \ - echo " FAILURE"; \ - tail -12 $LOGFILE | sed -e "s/^/[NODE] /g"; \ - echo "[NODE] ---------------------------------------------------------------------------"; \ - echo "FAILURE ec=$RETVAL" >>$LOGFILE; \ - else \ - echo " SUCCESS"; \ - tail -5 $LOGFILE | sed -e "s/^/[NODE] /g"; \ - echo "[NODE] ---------------------------------------------------------------------------"; \ - echo "SUCCESS" >>$LOGFILE; \ - fi; \ - exit $RETVAL' - - failures=$(egrep '^FAILURE' logs/*.out.txt | sed -e 's/^logs\///' -e 's/\.out\.txt:.*//' -e 's/^/ /') - successes=$(egrep '^SUCCESS' logs/*.out.txt | sed -e 's/^logs\///' -e 's/\.out\.txt:.*//' -e 's/^/ /') - - echo - echo "Failures:" - echo '------------------' - echo "$failures" - echo - echo "Successes:" - echo '------------------' - echo "$successes" -} - -## -- main -- ## - -# cleanup old logs -mkdir logs >/dev/null 2>&1 -rm -f logs/* - -# spin up vms sequentially, because openstack provider doesn't support --parallel -# This step will update `/etc/hosts` file in vms, because since version 1.5 vagrant up runs hostmanager before provision -echo ' ==> Calling "vagrant up" to boot the vms...' -vagrant up --no-provision - -# but run provision tasks in parallel -echo " ==> Beginning parallel 'vagrant provision' processes ..." -for ((i=1; i<=$NUM_INSTANCE; i++));do - cat <<EOF -hadoop-bigtop$i -EOF -done | parallel_provision - -#run smoketest on the last node when all node finish provisioning -echo "preparing for smoke tests..." -if [ "$RUN_SMOKE_TESTS" = "true" ]; then - echo "running smoke tests..." - vagrant ssh hadoop-bigtop-para$NUM_INSTANCE -c "sudo su <<HERE - cd /bigtop-home/bigtop-tests/smoke-tests - export HADOOP_CONF_DIR=/etc/hadoop/conf/ - export BIGTOP_HOME=/bigtop-home/ - export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/ - export HIVE_HOME=/usr/lib/hive/ - export PIG_HOME=/usr/lib/pig/ - export FLUME_HOME=/usr/lib/flume/ - export HIVE_CONF_DIR=/etc/hive/conf/ - export JAVA_HOME=/usr/lib/jvm/java-openjdk/ - export MAHOUT_HOME=/usr/lib/mahout - export ITEST="0.7.0" - - su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root' - su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant' - su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root' - - yum install -y pig hive flume mahout sqoop - - ./gradlew clean compileGroovy test -Dsmoke.tests=${SMOKE_TEST_COMPONENTS} --info - HERE" > logs/smoke.tmp 2>&1 - sed "s,\x1B\[[0-9;]*[a-zA-Z],,g" logs/smoke.tmp | tr -d '^M' > logs/smoke_tests.log -else - echo "Smoke tests did not run because run_smoke_tests set to false" -fi - -rm -f logs/*.tmp http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-openstack/vagrantconfig.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-openstack/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet-openstack/vagrantconfig.yaml deleted file mode 100644 index 86077dd..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-openstack/vagrantconfig.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/6/x86_64"; -num_instances: 1 -distro: centos -components: [hadoop, yarn] -enable_local_repo: false -run_smoke_tests: false -run_in_parallel: false -smoke_test_components: [mapreduce, pig] -jdk: "java-1.7.0-openjdk-devel.x86_64" -flavor: "" -image_id: "" -keypair_name: "" -key_path: "" -FQDN: "" http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-vm/.gitignore ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/.gitignore b/bigtop-deploy/vm/vagrant-puppet-vm/.gitignore deleted file mode 100644 index 04204c7..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-vm/.gitignore +++ /dev/null @@ -1 +0,0 @@ -config http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-vm/README.md ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/README.md b/bigtop-deploy/vm/vagrant-puppet-vm/README.md deleted file mode 100644 index 0090313..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-vm/README.md +++ /dev/null @@ -1,107 +0,0 @@ - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - ------------------------------------------------------------------------------------------------------------------------------------------------------- - -#BigTop VM provisioner - -## Overview - -The Vagrantfile definition creates a bigtop virtual hadoop cluster for you, by pulling from existing publishing bigtop repositories. -This cluster can be used: - -- to test bigtop smoke tests -- to test bigtop puppet recipes - -Eventually, we may also add ability to build AND provision bigtop in a vagrant recipe, which would essentially -give full validation of the BigTop stack. - -## USAGE - -1) Install [Vagrant](https://www.vagrantup.com/downloads.html)(DO NOT install 1.8.5 because of a critical [bug](https://github.com/mitchellh/vagrant/issues/7631)) - -2) Install [vagrant-hostmanager plugin](https://github.com/smdahlen/vagrant-hostmanager) to manage `/etc/hosts` - -``` -vagrant plugin install vagrant-hostmanager -``` - -3) (Optional) Install [vagrant-cachier plugin](https://github.com/fgrehm/vagrant-cachier) to cache packages at local - -``` -vagrant plugin install vagrant-cachier -``` - -4) To provision a one node Apache Hadoop cluster on top of vagrant boxes - -``` -vagrant up -``` - -5) You can specify number of nodes you'd like to provision by modifying `num_instances` in vagrantconfig.yaml - -``` -num_instances: 5 -``` - -6) Test on local built packages is available by: - -first, build up local yum repo - -``` -cd bigtop; ./gradlew alluxio-rpm; ./gradlew yum -``` - -and then enable local yum in vagrantconfig.yaml - -``` -enable_local_repo: true -``` - -finally, spin up the cluster -``` -vagrant up -``` - -## Override Vagrant configurations -You can override vagrant configurations from environment variables: -``` -$ export REPO=http://repo.example.com/path/to -$ vagrant up -``` - -or - -``` -$ MEMORY_SIZE=8000 vagrant up -``` - -##Configure Apache Hadoop ecosystem components -* Choose the ecosystem you want to be deployed by modifying components in vagrantconfig.yaml - -``` -components: [hadoop, hbase, yarn, ...] -``` - -By default, Apache Hadoop and YARN will be installed. -See `bigtop-deploy/puppet/hieradata/site.yaml` for more details. - -##Example: - -Run hbase-test.sh to evaluate the deployment. - -``` -../utils/hbase-test.sh -``` http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile deleted file mode 100755 index df36db2..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile +++ /dev/null @@ -1,161 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require "yaml" - -_config = YAML.load(File.open(File.join(File.dirname(__FILE__), "vagrantconfig.yaml"), File::RDONLY).read) -CONF = _config - -# Override vagrant configurations using environment variables -keys = CONF.keys -keys.each do |k| - if ENV[k.upcase] != nil then - puts "Overide from environment variable: " + k.upcase + " = " + ENV[k.upcase] - if /^\d+/.match(ENV[k.upcase]) - CONF[k] = Integer(ENV[k.upcase]) - else - CONF[k] = ENV[k.upcase] - end - end -end - -# Repository -# Example for testing a Release candidate. -repo = CONF['repo'] -# repo = "http://bigtop.s3.amazonaws.com/releases/0.7.0/redhat/6/x86_64"; - -# Which Linux Distribution to use. Right now only centos is tested -distro = CONF['distro'] - -# number of instances -num_instances = CONF['num_instances'] - -# hadoop ecosystem components -components = CONF['components'] - -# Whether to run smoke tests -run_smoke_tests = CONF['run_smoke_tests'] - -# Smoke test Components to run -smoke_test_components = CONF['smoke_test_components'].join(',') - -# This is a update to allow dev packages -# Force success - not worried if this step fails, since we generally only use it for development. -enable_local_repo = CONF['enable_local_repo'] -puts "vagrant conf local repo enabled: #{enable_local_repo}" - -# JDK package name -jdk = CONF['jdk'] - -# master node hostname -bigtop_master = "bigtop1.vagrant" - -$script = <<SCRIPT -service iptables stop -service firewalld stop -chkconfig iptables off -# Remove 127.0.0.1 entry since vagrant's hostname setting will map it to FQDN, -# which miss leads some daemons to bind on 127.0.0.1 instead of public or private IP -sed -i "/127.0.0.1/d" /etc/hosts -echo "Bigtop yum repo = #{repo}" -# Prepare host manipulation files needed in standalone box -cp /bigtop-home/bigtop-deploy/vm/vagrant-puppet-vm/config_hosts /etc/init.d -cp /bigtop-home/bigtop-deploy/vm/vagrant-puppet-vm/gen_hosts.sh /usr/bin -chkconfig --add config_hosts -# Prepare puppet configuration file -mkdir -p /etc/puppet/hieradata -cp /bigtop-home/bigtop-deploy/puppet/hiera.yaml /etc/puppet -cp -r /bigtop-home/bigtop-deploy/puppet/hieradata/bigtop/ /etc/puppet/hieradata/ -cat > /etc/puppet/hieradata/site.yaml << EOF -bigtop::hadoop_head_node: #{bigtop_master} -hadoop::hadoop_storage_dirs: [/data/1, /data/2] -bigtop::bigtop_repo_uri: #{repo} -hadoop_cluster_node::cluster_components: #{components} -bigtop::jdk_package_name: #{jdk} -hadoop::common_hdfs::testonly_hdfs_sshkeys: "yes" -EOF -SCRIPT - -VAGRANTFILE_API_VERSION = "2" -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - # manage /etc/hosts by hostmanager plugin(https://github.com/smdahlen/vagrant-hostmanager) - # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier) - config.hostmanager.enabled = true - - # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier) - if Vagrant.has_plugin?("vagrant-cachier") - config.cache.scope = :box - end - - # nodes definition - (1..num_instances).each do |i| - config.vm.define "bigtop#{i}" do |bigtop| - - bigtop.vm.box = CONF['box'] - bigtop_hostname="bigtop#{i}" - bigtop_fqdn="#{bigtop_hostname}.vagrant" - bigtop_ip="10.10.10.1#{i}" - - bigtop.vm.provider :virtualbox do |vb| - vb.customize ["modifyvm", :id, "--memory", CONF['memory_size']] - vb.customize ['modifyvm', :id, '--cpus', CONF['number_cpus']] - end - - bigtop.vm.network :private_network, ip: bigtop_ip - bigtop.vm.hostname = bigtop_fqdn - - # three levels up is the bigtop "home" directory. - # the current directory has puppet recipes which we need for provisioning. - bigtop.vm.synced_folder "../../../", "/bigtop-home" - - # We also add the bigtop-home output/ dir, so that locally built rpms will be available. - puts "Adding output/ repo ? #{enable_local_repo}" - - # carry on w/ installation - bigtop.vm.provision :shell do |shell| - shell.path = "../utils/setup-env-" + distro + ".sh" - shell.args = ["#{enable_local_repo}"] - end - bigtop.vm.provision "shell", inline: $script - # Add the ip to FQDN and hostname mapping in /etc/hosts - bigtop.hostmanager.aliases = "#{bigtop_fqdn} #{bigtop_hostname}" - - # deploy Apache Hadoop and it's ecosystem - bigtop.vm.provision :puppet do |puppet| - puppet.module_path = "../../puppet/modules/" - puppet.manifests_path = "../../puppet/manifests/" - puppet.manifest_file = "." - puppet.options = "--parser future" - end - - if run_smoke_tests then - if i==num_instances then - puts "creating provisioner directive for running tests" - bigtop.vm.provision :shell do |shell| - shell.path = "../utils/smoke-tests.sh" - shell.args = ["#{smoke_test_components}"] - end - else - puts "Not creating provisioner directive for tests yet... only on vm #{i} of #{num_instances}" - end - end - - end - end - -end http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-vm/config_hosts ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/config_hosts b/bigtop-deploy/vm/vagrant-puppet-vm/config_hosts deleted file mode 100755 index c584a78..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-vm/config_hosts +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# Hosts reconfiguration script -# chkconfig: 345 70 1 -# description: config_hosts - -# Licensed to Hortonworks, Inc. under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. Hortonworks, Inc. licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -NAME="config_hosts" -LOG="/var/log/config_hosts.log" - -source /etc/rc.d/init.d/functions - -case "$1" in -start) - printf "%-50s\n" "Starting $NAME..." | tee -a $LOG - - printf "%-50s\n" "Updating IP..." | tee -a $LOG - bash /usr/bin/gen_hosts.sh - -;; -stop) - date >> $LOG -;; - -restart) -;; - -*) - echo "Usage: $0 {start|stop|restart}" - exit 1 -esac http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-vm/gen_hosts.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/gen_hosts.sh b/bigtop-deploy/vm/vagrant-puppet-vm/gen_hosts.sh deleted file mode 100755 index c629c85..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-vm/gen_hosts.sh +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to Hortonworks, Inc. under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. Hortonworks, Inc. licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -echo "# Do not remove the following line, or various programs" > /etc/hosts -echo "# that require network functionality will fail." >> /etc/hosts -echo "127.0.0.1 localhost.localdomain localhost" >> /etc/hosts - -function get_inet_iface(){ - route | grep default | awk '{print $8}' -} - -function get_ip() { - ip addr | grep 'inet ' | grep -E "($(get_inet_iface))" | awk '{ print $2 }' | awk -F'/' '{print $1}' -} - -HOST=$(get_ip) -NUM=5 -while [ -z "$HOST" ]; do - HOST=$(get_ip) - sleep 5 - NUM=$(($NUM-1)) - if [ $NUM -le 0 ]; then - HOST="127.0.0.1" - echo "Failed to update IP" - break - fi -done -echo "$HOST `hostname`" >> /etc/hosts - -echo 0 > /proc/sys/kernel/hung_task_timeout_secs -ethtool -K $(get_inet_iface) tso off http://git-wip-us.apache.org/repos/asf/bigtop/blob/4ee8f075/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml deleted file mode 100644 index 803e4d3..0000000 --- a/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -memory_size: 4096 -number_cpus: 1 -box: "puppetlabs/centos-7.2-64-nocm" -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64"; -num_instances: 1 -distro: centos -components: [hadoop, yarn] -enable_local_repo: false -run_smoke_tests: false -smoke_test_components: [mapreduce, pig] -jdk: "java-1.7.0-openjdk-devel.x86_64"
