Repository: bigtop Updated Branches: refs/heads/master eff797efd -> 3f64dfdcc
BIGTOP-1562. Critical tachyon puppet improvements Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/3f64dfdc Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/3f64dfdc Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/3f64dfdc Branch: refs/heads/master Commit: 3f64dfdcc88d25adbc1de4c3b2eadd0b7dbeca9f Parents: eff797e Author: jayunit100 <[email protected]> Authored: Sat Dec 6 12:08:16 2014 -0500 Committer: jayunit100 <[email protected]> Committed: Sun Dec 7 19:09:19 2014 -0500 ---------------------------------------------------------------------- bigtop-deploy/puppet/manifests/cluster.pp | 13 +++- .../puppet/modules/tachyon/manifests/init.pp | 29 ++++++++- .../modules/tachyon/templates/log4j.properties | 41 +++++++++++++ .../modules/tachyon/templates/tachyon-env.sh | 64 ++++++++++++++++++++ bigtop-deploy/vm/vagrant-puppet/Vagrantfile | 9 +++ .../vm/vagrant-puppet/vagrantconfig.yaml | 6 +- 6 files changed, 153 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/3f64dfdc/bigtop-deploy/puppet/manifests/cluster.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index e7f868f..a2731f5 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -100,6 +100,7 @@ class hadoop_cluster_node { $giraph_zookeeper_quorum = $hadoop_head_node $spark_master_host = $hadoop_head_node + $tachyon_master_host = $hadoop_head_node $hadoop_zookeeper_ensemble = ["$hadoop_head_node:2888:3888"] @@ -194,6 +195,13 @@ class hadoop_worker_node inherits hadoop_cluster_node { master_ui_port => $spark_master_ui_port, } } + + if ($components[0] == undef or "tachyon" in $components) { + tachyon::worker { "tachyon worker": + master_host => $tachyon_master_host + } + } + } class hadoop_head_node inherits hadoop_worker_node { @@ -291,9 +299,8 @@ if ($hadoop_security_authentication == "kerberos") { } if ($components[0] == undef or "tachyon" in $components) { - tachyon::master { "tachyon master": - master_host => $tachyon_master_host, - master_port => $tachyon_master_port + tachyon::master { "tachyon-master": + master_host => $tachyon_master_host } } http://git-wip-us.apache.org/repos/asf/bigtop/blob/3f64dfdc/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp b/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp index 8af9b68..55fb34a 100644 --- a/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp @@ -14,11 +14,30 @@ class tachyon { package { "tachyon": ensure => latest, } + + # add logging into /var/log/.. + file { + "/etc/tachyon/log4j.properties": + content => template("tachyon/log4j.properties"), + require => [Package["tachyon"]] + } + + # add tachyon-env.sh to point to tachyon master + file { "/etc/tachyon/tachyon-env.sh": + content => template("tachyon/tachyon-env.sh"), + require => [Package["tachyon"]] + } } - define master($master_host, $master_port) { + define master($master_host) { include common + exec { + "tachyon formatting": + command => "/usr/lib/tachyon/bin/tachyon format", + require => [ Package["tachyon"]] + } + if ( $fqdn == $master_host ) { service { "tachyon-master": ensure => running, @@ -27,17 +46,21 @@ class tachyon { hasstatus => true, } } + } - define worker($master_host, $master_port) { + define worker($master_host) { include common if ( $fqdn == $master_host ) { + notice("tachyon ---> master host") # We want master to run first in all cases Service["tachyon-master"] ~> Service["tachyon-worker"] - } + } + service { "tachyon-worker": ensure => running, + require => [Package["tachyon"]], hasrestart => true, hasstatus => true, } http://git-wip-us.apache.org/repos/asf/bigtop/blob/3f64dfdc/bigtop-deploy/puppet/modules/tachyon/templates/log4j.properties ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/tachyon/templates/log4j.properties b/bigtop-deploy/puppet/modules/tachyon/templates/log4j.properties new file mode 100644 index 0000000..ae84434 --- /dev/null +++ b/bigtop-deploy/puppet/modules/tachyon/templates/log4j.properties @@ -0,0 +1,41 @@ +# May get overridden by System Property +tachyon.logger.type=Console + +log4j.rootLogger=INFO, ${tachyon.logger.type} + +log4j.appender.Console=org.apache.log4j.ConsoleAppender +log4j.appender.Console.Target=System.out +log4j.appender.Console.layout=org.apache.log4j.PatternLayout +log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} (%F:%M) - %m%n + +# Appender for Master +log4j.appender.MASTER_LOGGER=tachyon.Log4jFileAppender +log4j.appender.MASTER_LOGGER.File=/var/log/tachyon/master.log + +log4j.appender.MASTER_LOGGER.MaxFileSize=10 +log4j.appender.MASTER_LOGGER.MaxBackupIndex=100 +log4j.appender.MASTER_LOGGER.DeletionPercentage=10 +log4j.appender.MASTER_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n +#log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + +# Appender for Workers +log4j.appender.WORKER_LOGGER=tachyon.Log4jFileAppender +log4j.appender.WORKER_LOGGER.File=/var/log/tachyon/worker.log + +log4j.appender.WORKER_LOGGER.MaxFileSize=10 +log4j.appender.WORKER_LOGGER.MaxBackupIndex=100 +log4j.appender.WORKER_LOGGER.DeletionPercentage=10 +log4j.appender.WORKER_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n +#log4j.appender.WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + +# Appender for User +log4j.appender.USER_LOGGER=tachyon.Log4jFileAppender +log4j.appender.USER_LOGGER.File=/var/log/tachyon/logs/user.log +log4j.appender.USER_LOGGER.MaxFileSize=10 +log4j.appender.USER_LOGGER.MaxBackupIndex=10 +log4j.appender.USER_LOGGER.DeletionPercentage=20 +log4j.appender.USER_LOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.USER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n +#log4j.appender.USER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n http://git-wip-us.apache.org/repos/asf/bigtop/blob/3f64dfdc/bigtop-deploy/puppet/modules/tachyon/templates/tachyon-env.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/tachyon/templates/tachyon-env.sh b/bigtop-deploy/puppet/modules/tachyon/templates/tachyon-env.sh new file mode 100755 index 0000000..1094976 --- /dev/null +++ b/bigtop-deploy/puppet/modules/tachyon/templates/tachyon-env.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# This file contains environment variables required to run Tachyon. Copy it as tachyon-env.sh and +# edit that to configure Tachyon for your site. At a minimum, +# the following variables should be set: +# +# - JAVA_HOME, to point to your JAVA installation +# - TACHYON_MASTER_ADDRESS, to bind the master to a different IP address or hostname +# - TACHYON_UNDERFS_ADDRESS, to set the under filesystem address. +# - TACHYON_WORKER_MEMORY_SIZE, to set how much memory to use (e.g. 1000mb, 2gb) per worker +# - TACHYON_RAM_FOLDER, to set where worker stores in memory data +# - TACHYON_UNDERFS_HDFS_IMPL, to set which HDFS implementation to use (e.g. com.mapr.fs.MapRFileSystem, +# org.apache.hadoop.hdfs.DistributedFileSystem) + +# The following gives an example: + +if [[ `uname -a` == Darwin* ]]; then + # Assuming Mac OS X + export JAVA_HOME=${JAVA_HOME:-$(/usr/libexec/java_home)} + export TACHYON_RAM_FOLDER=/Volumes/ramdisk + export TACHYON_JAVA_OPTS="-Djava.security.krb5.realm= -Djava.security.krb5.kdc=" +else + # Assuming Linux + if [ -z "$JAVA_HOME" ]; then + export JAVA_HOME=/usr/lib/jvm/java-7-oracle + fi + export TACHYON_RAM_FOLDER=/mnt/ramdisk +fi + +export JAVA="$JAVA_HOME/bin/java" + +echo "Starting tachyon w/ java = $JAVA " + +export TACHYON_MASTER_ADDRESS=<%= @tachyon_master_address %> +export TACHYON_UNDERFS_ADDRESS=$TACHYON_HOME/underfs +#export TACHYON_UNDERFS_ADDRESS=hdfs://localhost:9000 +export TACHYON_WORKER_MEMORY_SIZE=1GB +export TACHYON_UNDERFS_HDFS_IMPL=org.apache.hadoop.hdfs.DistributedFileSystem + +echo "TACHYON master => $TACHYON_MASTER_ADDRESS " + +CONF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +export TACHYON_JAVA_OPTS+=" + -Dlog4j.configuration=file:$CONF_DIR/log4j.properties + -Dtachyon.debug=false + -Dtachyon.underfs.address=$TACHYON_UNDERFS_ADDRESS + -Dtachyon.underfs.hdfs.impl=$TACHYON_UNDERFS_HDFS_IMPL + -Dtachyon.data.folder=$TACHYON_UNDERFS_ADDRESS/tmp/tachyon/data + -Dtachyon.workers.folder=$TACHYON_UNDERFS_ADDRESS/tmp/tachyon/workers + -Dtachyon.worker.memory.size=$TACHYON_WORKER_MEMORY_SIZE + -Dtachyon.worker.data.folder=$TACHYON_RAM_FOLDER/tachyonworker/ + -Dtachyon.master.worker.timeout.ms=60000 + -Dtachyon.master.hostname=$TACHYON_MASTER_ADDRESS + -Dtachyon.master.journal.folder=$TACHYON_HOME/journal/ + -Dorg.apache.jasper.compiler.disablejsr199=true + -Djava.net.preferIPv4Stack=true +" + +# Master specific parameters. Default to TACHYON_JAVA_OPTS. +export TACHYON_MASTER_JAVA_OPTS="$TACHYON_JAVA_OPTS" + +# Worker specific parameters that will be shared to all workers. Default to TACHYON_JAVA_OPTS. +export TACHYON_WORKER_JAVA_OPTS="$TACHYON_JAVA_OPTS" http://git-wip-us.apache.org/repos/asf/bigtop/blob/3f64dfdc/bigtop-deploy/vm/vagrant-puppet/Vagrantfile ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile index d5eca4a..05a66c2 100755 --- a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile +++ b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile @@ -66,6 +66,15 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # the current directory has puppet recipes which we need for provisioning. bigtop.vm.synced_folder "../../../", "/bigtop-home" + # We also add the bigtop-home output/ dir, so that locally built rpms will be available. + puts "Adding rpms from /output/ for local testing." + + # This is a update to allow dev packages + # Force success - not worried if this step fails, since we generally only use it for development. + bigtop.vm.provision "shell", inline: "sudo echo \"gpgcheck=0\" >> /etc/yum.conf" + bigtop.vm.provision "shell", inline: "sudo yum-config-manager --add-repo file:///bigtop-home/output ; true" + + # carry on w/ installation bigtop.vm.provision "shell", path: "../utils/setup-env.sh" # shell provisioner to setup basic vm environment http://git-wip-us.apache.org/repos/asf/bigtop/blob/3f64dfdc/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml index d045ce2..4306dbb 100644 --- a/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml +++ b/bigtop-deploy/vm/vagrant-puppet/vagrantconfig.yaml @@ -1,6 +1,6 @@ -memory_size: 1024 +memory_size: 6024 number_cpus: 1 box: "puppetlabs/centos-6.5-64-nocm" repo: "http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/"; -num_instances: 3 -components: "hadoop,hbase,yarn,mapred-app" +num_instances: 1 +components: "hadoop,yarn,mapred-app,tachyon"
