Repository: bigtop Updated Branches: refs/heads/master c8a61d98f -> 940b6fda2
BIGTOP-1556. Cleanup some unused minor stuff. Signed-off-by: Konstantin Boudnik <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/4a7b61b9 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/4a7b61b9 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/4a7b61b9 Branch: refs/heads/master Commit: 4a7b61b9ce4b17ce46429505d8328f82a3a65042 Parents: c8a61d9 Author: Dasha Boudnik <[email protected]> Authored: Thu Jan 8 10:57:13 2015 -0800 Committer: Konstantin Boudnik <[email protected]> Committed: Thu Jan 8 10:57:13 2015 -0800 ---------------------------------------------------------------------- DEVNOTES | 25 -- bigtop-deploy/live-cd/apache-bigtop.ks | 245 -------------------- bigtop-deploy/vm/boxgrinder/bigtop-init-hdfs | 64 ----- bigtop-deploy/vm/boxgrinder/bigtop_hadoop.appl | 48 ---- bigtop-deploy/vm/vagrant/README | 11 - bigtop-deploy/vm/vagrant/Vagrantfile | 53 ----- bigtop-deploy/vm/vagrant/provision.sh | 45 ---- 7 files changed, 491 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a7b61b9/DEVNOTES ---------------------------------------------------------------------- diff --git a/DEVNOTES b/DEVNOTES deleted file mode 100644 index f94f2a9..0000000 --- a/DEVNOTES +++ /dev/null @@ -1,25 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -1. Debian - 1.0. apt-get install -y git subversion build-essential dh-make debhelper devscripts ant ant-optional autoconf automake subversion liblzo2-dev libzip-dev sharutils libfuse-dev reprepro libssl-dev - - FIXME?: You also need to have Scala 2.9.3 installed and SCALA_HOME - env. var pointed properly. Otherwise, Spark project won't be build as it - requires scala. Only older version of the language environment is available - on Ubuntu at the moment. - -2. RedHat - 2.0 yum install -y git subversion fuse-devel fuse fuse-libs http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a7b61b9/bigtop-deploy/live-cd/apache-bigtop.ks ---------------------------------------------------------------------- diff --git a/bigtop-deploy/live-cd/apache-bigtop.ks b/bigtop-deploy/live-cd/apache-bigtop.ks deleted file mode 100644 index 2c51eb2..0000000 --- a/bigtop-deploy/live-cd/apache-bigtop.ks +++ /dev/null @@ -1,245 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -install - -repo --name=fedora --mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=fedora-$releasever&arch=$basearch -repo --name=updates --mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=updates-released-f$releasever&arch=$basearch - -#trunk -repo --name=bigtop --baseurl=http://bigtop01.cloudera.org:8080/job/Bigtop-trunk-Repository/label=fedora16/lastSuccessfulBuild/artifact/repo/ --noverifyssl - -#RCs -#repo --name=bigtop --baseurl=http://bigtop01.cloudera.org:8080/job/Bigtop-RCs-Repository/./label=fedora16//lastSuccessfulBuild/artifact/repo/ - -lang en_US.UTF-8 -keyboard us -rootpw --plaintext bigtop -firewall --disabled -authconfig --enableshadow --enablemd5 -selinux --disabled -timezone --utc America/Los_Angeles -xconfig --startxonboot -part / --size 4096 --fstype ext4 - -%packages -@base -@base-x -@core -@java -@kde-desktop -@admin-tools -@graphical-internet -@hardware-support -@fonts -@input-methods -#@development-tools -#@development-libs -#@engineering-and-scientific -#-gnome-* --verne-backgrounds-gnome --transmission-* --opencv --marble-* --marble --kdegames-* --k3b* --akonadi* --scribus -grub-efi -grub2 -efibootmgr - -vim-enhanced -vim-X11 - -hadoop -hadoop-conf-pseudo -hbase -hbase-master -hbase-doc -hbase-regionserver -hbase-thrift -hive -hive-metastore -hive-server -sqoop -sqoop-metastore -flume -flume-agent -mahout -pig -oozie -oozie-client -zookeeper -zookeeper-server -%end -%post - -/usr/sbin/adduser --create-home bigtop -passwd -d bigtop - -echo 'install ipv6 /bin/true' >> /etc/modprobe.d/disable-ipv6.conf -echo 'bigtop ALL=NOPASSWD: ALL' >> /etc/sudoers - -cat > /etc/sysconfig/desktop <<EOF -DESKTOP="KDE" -DISPLAYMANAGER="KDE" -EOF - -cat > /home/bigtop/.dmrc <<EOF -[Desktop] -Language=en_US.utf8 -Layout=us -Session=kde-plasma -EOF -/bin/chmod 644 /home/bigtop/.dmrc -/bin/chown bigtop:bigtop /home/bigtop/.dmrc - -sed -ie 's/#AutoLoginEnable=true/AutoLoginEnable=true/' /etc/kde/kdm/kdmrc -sed -ie 's/#AutoLoginUser=.*/AutoLoginUser=bigtop/' /etc/kde/kdm/kdmrc -sed -i 's/#PreselectUser=Default/PreselectUser=Default/' /etc/kde/kdm/kdmrc -sed -i 's/#DefaultUser=.*/DefaultUser=bigtop/' /etc/kde/kdm/kdmrc - -mkdir -p /home/bigtop/.kde/share/config/ -cat > /home/bigtop/.kde/share/config/nepomukserverrc <<EOF -[Basic Settings] -Start Nepomuk=false -[Service-nepomukstrigiservice] -autostart=false -EOF - -mkdir -p /home/bigtop/.kde/share/config/ -cat > /home/bigtop/.kde/share/config/apper <<EOF -[CheckUpdate] -autoUpdate=0 -interval=0 -EOF - -mkdir -p /home/bigtop/Desktop -cat > /home/bigtop/Desktop/HDFS <<EOF -#!/usr/bin/env xdg-open -[Desktop Entry] -Comment[en_US]= -Comment= -Exec=/usr/bin/firefox http://localhost:50070/ -GenericName[en_US]=HDFS -GenericName=HDFS -MimeType= -Name[en_US]=HDFS -Name=HDFS -Path= -StartupNotify=false -Terminal=false -TerminalOptions= -Type=Application -X-DBUS-ServiceName= -X-DBUS-StartupType= -X-KDE-SubstituteUID=false -X-KDE-Username= -EOF - -cat > /home/bigtop/Desktop/MapReduce <<EOF -#!/usr/bin/env xdg-open -[Desktop Entry] -Comment[en_US]= -Comment= -Exec=/usr/bin/firefox http://localhost:50030/ -GenericName[en_US]=MapReduce -GenericName=MapReduce -MimeType= -Name[en_US]=MapReduce -Name=MapReduce -Path= -StartupNotify=false -Terminal=false -TerminalOptions= -Type=Application -X-DBUS-ServiceName= -X-DBUS-StartupType= -X-KDE-SubstituteUID=false -X-KDE-Username= -EOF - -cat > /home/bigtop/Desktop/HBase <<EOF -#!/usr/bin/env xdg-open -[Desktop Entry] -Comment[en_US]= -Comment= -Exec=/usr/bin/firefox http://localhost:60010/ -GenericName[en_US]=HBase -GenericName=HBase -MimeType= -Name[en_US]=HBase -Name=HBase -Path= -StartupNotify=false -Terminal=false -TerminalOptions= -Type=Application -X-DBUS-ServiceName= -X-DBUS-StartupType= -X-KDE-SubstituteUID=false -X-KDE-Username= -EOF - -cat > /home/bigtop/Desktop/Oozie <<EOF -#!/usr/bin/env xdg-open -[Desktop Entry] -Comment[en_US]= -Comment= -Exec=/usr/bin/firefox http://localhost:11000/ -GenericName[en_US]=Oozie -GenericName=Oozie -MimeType= -Name[en_US]=Oozie -Name=Oozie -Path= -StartupNotify=false -Terminal=false -TerminalOptions= -Type=Application -X-DBUS-ServiceName= -X-DBUS-StartupType= -X-KDE-SubstituteUID=false -X-KDE-Username= -EOF - - -cat > /etc/hbase/conf/hbase-site.xml <<EOF -<configuration> - <property> - <name>hbase.cluster.distributed</name> - <value>true</value> - </property> - <property> - <name>hbase.zookeeper.quorum</name> - <value>localhost</value> - </property> - <property> - <name>hbase.rootdir</name> - <value>hdfs://localhost:8020/hbase</value> - </property> -</configuration> -EOF - -chown -R bigtop:bigtop /home/bigtop/ - -systemctl disable firstboot-text.service -systemctl disable firstboot-graphical.service - -/usr/bin/yes Y | su hdfs /bin/bash -c '/usr/bin/hadoop namenode -format' -%end http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a7b61b9/bigtop-deploy/vm/boxgrinder/bigtop-init-hdfs ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/boxgrinder/bigtop-init-hdfs b/bigtop-deploy/vm/boxgrinder/bigtop-init-hdfs deleted file mode 100755 index b0295da..0000000 --- a/bigtop-deploy/vm/boxgrinder/bigtop-init-hdfs +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Initialize HDFS -# -# chkconfig: 345 90 15 -# description: Hadoop nodemanager -# -### BEGIN INIT INFO -# Provides: bigtop-init-hdfs -# Short-Description: Initialize HDFS directories used by Yarn -# Default-Start: 3 4 5 -# Default-Stop: 0 1 2 6 -# Required-Start: $syslog $remote_fs hadoop-hdfs-namenode -# Required-Stop: $syslog $remote_fs -# Should-Start: -# Should-Stop: -### END INIT INFO - -. /lib/lsb/init-functions - -# Ensure this is run only once -FIRST_EXECUTION_LOCK="/var/lib/bigtop-init-hdfs" - -RETVAL=0 - -start() { - log_success_msg "Initializing HDFS" - - if [ ! -e "${FIRST_EXECUTION_LOCK}" ]; then - /usr/lib/hadoop/libexec/init-hdfs.sh - touch "${FIRST_EXECUTION_LOCK}" - fi - RETVAL=$? - return $RETVAL -} - -service() { - case "$1" in - start) - start - ;; - *) - exit 0 - esac -} - -service "$1" - -exit $RETVAL http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a7b61b9/bigtop-deploy/vm/boxgrinder/bigtop_hadoop.appl ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/boxgrinder/bigtop_hadoop.appl b/bigtop-deploy/vm/boxgrinder/bigtop_hadoop.appl deleted file mode 100644 index 7dbbbe9..0000000 --- a/bigtop-deploy/vm/boxgrinder/bigtop_hadoop.appl +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: bigtop_hadoop -summary: Apache Bigtop VM with Apache Hadoop -version: 2 -release: 0 -os: - name: centos - version: 6 - password: bigtop -hardware: - partitions: - "/": - size: 2 - type: ext4 -repos: - - name: "Bigtop" - baseurl: "http://bigtop01.cloudera.org:8080/view/Bigtop-trunk/job/Bigtop-trunk-Repository/label=centos6/lastSuccessfulBuild/artifact/repo/"; -packages: - - @core - - shadow-utils - - passwd - - screen - - sed - - vim-enhanced - - wget - - java-1.6.0-openjdk-devel - - hadoop-conf-pseudo -files: - "/etc/init.d": - - "bigtop-init-hdfs" -post: - base: - - "/etc/init.d/hadoop-hdfs-namenode init" - - "/sbin/chkconfig bigtop-init-hdfs on" http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a7b61b9/bigtop-deploy/vm/vagrant/README ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant/README b/bigtop-deploy/vm/vagrant/README deleted file mode 100644 index 59b6d0d..0000000 --- a/bigtop-deploy/vm/vagrant/README +++ /dev/null @@ -1,11 +0,0 @@ -Set up a vagrant box with bigtop ecosystem. - -Shell example: - -1) Use fedora19 base box and - -2) yum install the repo - -2) yum-config-manager --add-repo http://bigtop01.cloudera.org:8080/view/Bigtop-trunk/job/Bigtop-trunk-repository/label=fedora18/lastSuccessfulBuild/artifact/repo/bigtop.repo - -3) yum install hadoop , ... http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a7b61b9/bigtop-deploy/vm/vagrant/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant/Vagrantfile b/bigtop-deploy/vm/vagrant/Vagrantfile deleted file mode 100644 index bc28eed..0000000 --- a/bigtop-deploy/vm/vagrant/Vagrantfile +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -VAGRANTFILE_API_VERSION = "2" -Vagrant.configure("2") do |config| - vmbox = "vagrant-fedora19B" - - # Virtual Box implementation - vmboxurl = "http://rhbd.s3-us-west-1.amazonaws.com/vagrant/fedora19B.box"; - - #First node in the cluster. - #HMaster - config.vm.define :bigtop1 do |bigtop1| - - # cachier plugin for faster install, - # see https://github.com/fgrehm/vagrant-cachier - # bigtop1.cache.enable :yum - - bigtop1.vm.box = vmbox - bigtop1.vm.box_url = vmboxurl - - bigtop1.vm.network :private_network, ip: "10.10.10.11" - - # bigtop1.vm.synced_folder "./config", "/vagrant" - - bigtop1.vm.hostname = "rs1" - bigtop1.vm.provision :shell, :path => "provision.sh" - end - - # Master node - # config.vm.define :bigtop2 do |bigtop2| - # bigtop2.cache.enable :yum - # bigtop2.cache.auto_detect = true - # bigtop2.vm.box = vmbox - # bigtop2.vm.box_url = vmboxurl - # bigtop2.vm.host_name = "master" - # bigtop2.vm.network "private_network", ip: "10.10.10.12" - # bigtop2.vm.synced_folder "./config", "/vagrant" - # bigtop2.vm.provision "shell", path: "provision.sh" - # end - -end http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a7b61b9/bigtop-deploy/vm/vagrant/provision.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant/provision.sh b/bigtop-deploy/vm/vagrant/provision.sh deleted file mode 100644 index 2aca938..0000000 --- a/bigtop-deploy/vm/vagrant/provision.sh +++ /dev/null @@ -1,45 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#Taken from : https://cwiki.apache.org/confluence/display/BIGTOP/How+to+install+Hadoop+distribution+from+Bigtop+0.6.0 -#A Vagrant recipe for setting up a hadoop box. - -#Get the apache yum repo -yum install -y wget java-1.7.0-openjdk-devel.x86_64 - -wget -O /etc/yum.repos.d/bigtop.repo http://www.apache.org/dist/bigtop/bigtop-0.6.0/repos/fedora18/bigtop.repo - -#Now install the base components -yum install -y hadoop\* mahout\* hive\* pig\* - -export JAVA_HOME=`sudo find /usr -name java-* | grep openjdk | grep 64 | grep "jvm/java" | grep -v fc` - -/etc/init.d/hadoop-hdfs-namenode init - -#Start each datanode -for i in hadoop-hdfs-namenode hadoop-hdfs-datanode ; - do service $i start ; -done - -/usr/lib/hadoop/libexec/init-hdfs.sh - -service hadoop-yarn-resourcemanager start -service hadoop-yarn-nodemanager start - -hadoop fs -ls -R / - -# Make a directory so that vagrant user has a dir to run jobs inside of. -su -s /bin/bash hdfs -c 'hadoop fs -mkdir /user/vagrant && hadoop fs -chown vagrant:vagrant /user/vagrant' - -su -s /bin/bash vagrant -c 'hadoop jar /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples*.jar pi 2 2'
