BIGTOP-2084: rename all puppet modules to have an underscore rather a dash
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/eab70662 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/eab70662 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/eab70662 Branch: refs/heads/master Commit: eab70662b03092383089007967afefd2647ac1f2 Parents: ae9dc84 Author: Olaf Flebbe <[email protected]> Authored: Thu Oct 8 13:23:57 2015 +0000 Committer: Olaf Flebbe <[email protected]> Committed: Wed Oct 14 21:52:11 2015 +0200 ---------------------------------------------------------------------- bigtop-deploy/puppet/README.md | 2 +- .../puppet/hieradata/bigtop/cluster.yaml | 22 +- bigtop-deploy/puppet/hieradata/site.yaml | 2 +- bigtop-deploy/puppet/manifests/cluster.pp | 16 +- .../modules/hadoop-flume/manifests/init.pp | 43 --- .../modules/hadoop-flume/templates/flume.conf | 36 -- .../puppet/modules/hadoop-flume/tests/init.pp | 44 --- .../modules/hadoop-hbase/manifests/init.pp | 118 ------ .../modules/hadoop-hbase/templates/hbase-env.sh | 82 ----- .../hadoop-hbase/templates/hbase-site.xml | 70 ---- .../modules/hadoop-hbase/templates/jaas.conf | 24 -- .../puppet/modules/hadoop-hbase/tests/init.pp | 29 -- .../modules/hadoop-hive/manifests/init.pp | 37 -- .../modules/hadoop-hive/templates/hive-site.xml | 68 ---- .../puppet/modules/hadoop-hive/tests/init.pp | 16 - .../modules/hadoop-oozie/manifests/init.pp | 74 ---- .../hadoop-oozie/templates/oozie-site.xml | 369 ------------------- .../puppet/modules/hadoop-oozie/tests/init.pp | 17 - .../puppet/modules/hadoop-pig/manifests/init.pp | 38 -- .../modules/hadoop-pig/templates/pig.properties | 70 ---- .../puppet/modules/hadoop-pig/tests/init.pp | 17 - .../modules/hadoop-zookeeper/files/java.env | 15 - .../modules/hadoop-zookeeper/manifests/init.pp | 95 ----- .../hadoop-zookeeper/templates/jaas.conf | 25 -- .../modules/hadoop-zookeeper/templates/zoo.cfg | 42 --- .../modules/hadoop-zookeeper/tests/init.pp | 20 - .../modules/hadoop_flume/manifests/init.pp | 43 +++ .../modules/hadoop_flume/templates/flume.conf | 36 ++ .../puppet/modules/hadoop_flume/tests/init.pp | 44 +++ .../modules/hadoop_hbase/manifests/init.pp | 118 ++++++ .../modules/hadoop_hbase/templates/hbase-env.sh | 82 +++++ .../hadoop_hbase/templates/hbase-site.xml | 70 ++++ .../modules/hadoop_hbase/templates/jaas.conf | 24 ++ .../puppet/modules/hadoop_hbase/tests/init.pp | 29 ++ .../modules/hadoop_hive/manifests/init.pp | 37 ++ .../modules/hadoop_hive/templates/hive-site.xml | 68 ++++ .../puppet/modules/hadoop_hive/tests/init.pp | 16 + .../modules/hadoop_oozie/manifests/init.pp | 74 ++++ .../hadoop_oozie/templates/oozie-site.xml | 369 +++++++++++++++++++ .../puppet/modules/hadoop_oozie/tests/init.pp | 17 + .../puppet/modules/hadoop_pig/manifests/init.pp | 38 ++ .../modules/hadoop_pig/templates/pig.properties | 70 ++++ .../puppet/modules/hadoop_pig/tests/init.pp | 17 + .../modules/hadoop_zookeeper/files/java.env | 15 + .../modules/hadoop_zookeeper/manifests/init.pp | 95 +++++ .../hadoop_zookeeper/templates/jaas.conf | 25 ++ .../modules/hadoop_zookeeper/templates/zoo.cfg | 42 +++ .../modules/hadoop_zookeeper/tests/init.pp | 20 + .../puppet/modules/hue/manifests/init.pp | 2 +- .../modules/ignite-hadoop/manifests/init.pp | 82 ----- .../ignite-hadoop/templates/core-site.xml | 65 ---- .../ignite-hadoop/templates/default-config.xml | 187 ---------- .../ignite-hadoop/templates/hive-site.xml | 33 -- .../ignite-hadoop/templates/ignite-hadoop | 21 -- .../ignite-hadoop/templates/mapred-site.xml | 63 ---- .../puppet/modules/ignite-hadoop/tests/init.pp | 16 - .../modules/ignite_hadoop/manifests/init.pp | 82 +++++ .../ignite_hadoop/templates/core-site.xml | 65 ++++ .../ignite_hadoop/templates/default-config.xml | 187 ++++++++++ .../ignite_hadoop/templates/hive-site.xml | 33 ++ .../ignite_hadoop/templates/ignite-hadoop | 21 ++ .../ignite_hadoop/templates/mapred-site.xml | 63 ++++ .../puppet/modules/ignite_hadoop/tests/init.pp | 16 + 63 files changed, 1838 insertions(+), 1838 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/README.md ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/README.md b/bigtop-deploy/puppet/README.md index 260d19c..6364b9e 100644 --- a/bigtop-deploy/puppet/README.md +++ b/bigtop-deploy/puppet/README.md @@ -131,7 +131,7 @@ hadoop::hadoop_storage_dirs: - "/data/1" - "/data/2" hadoop_cluster_node::cluster_components: - - ignite-hadoop + - ignite_hadoop - hive - spark - yarn http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml index 41d477b..e46ba1f 100644 --- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml +++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml @@ -106,11 +106,11 @@ bigtop::hadoop_zookeeper_port: "2181" hadoop::zk: "%{hiera('bigtop::hadoop_head_node')}:%{hiera('bigtop::hadoop_zookeeper_port')}" bigtop::hadoop_namenode_uri: "hdfs://%{hiera('hadoop::common_hdfs::hadoop_namenode_host')}:%{hiera('hadoop::common_hdfs::hadoop_namenode_port')}" -hadoop-hbase::base_relative_rootdir: "/hbase" -hadoop-hbase::common_config::rootdir: "%{hiera('bigtop::hadoop_namenode_uri')}%{hiera('hadoop-hbase::base_relative_rootdir')}" -hadoop-hbase::common_config::zookeeper_quorum: "%{hiera('bigtop::hadoop_head_node')}" -hadoop-hbase::common_config::kerberos_realm: "%{hiera('kerberos::site::realm')}" -hadoop-hbase::client::thrift: true +hadoop_hbase::base_relative_rootdir: "/hbase" +hadoop_hbase::common_config::rootdir: "%{hiera('bigtop::hadoop_namenode_uri')}%{hiera('hadoop_hbase::base_relative_rootdir')}" +hadoop_hbase::common_config::zookeeper_quorum: "%{hiera('bigtop::hadoop_head_node')}" +hadoop_hbase::common_config::kerberos_realm: "%{hiera('kerberos::site::realm')}" +hadoop_hbase::client::thrift: true solr::server::root_url: "%{hiera('bigtop::hadoop_namenode_uri')}" solr::server::zk: "%{hiera('hadoop::zk')}" @@ -118,7 +118,7 @@ solr::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" # Default but needed here to make sure, hue uses the same port solr::server::port: "1978" -hadoop-oozie::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" +hadoop_oozie::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" hcatalog::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" hcatalog::webhcat::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" @@ -127,10 +127,10 @@ spark::common::spark_master_host: "%{hiera('bigtop::hadoop_head_node')}" tachyon::common::master_host: "%{hiera('bigtop::hadoop_head_node')}" -hadoop-zookeeper::server::myid: "0" -hadoop-zookeeper::server::ensemble: +hadoop_zookeeper::server::myid: "0" +hadoop_zookeeper::server::ensemble: - "%{hiera('bigtop::hadoop_head_node')}:2888:3888" -hadoop-zookeeper::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" +hadoop_zookeeper::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" # those are only here because they were present as extlookup keys previously bigtop::hadoop_rm_http_port: "8088" @@ -157,7 +157,7 @@ hue::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" giraph::client::zookeeper_quorum: "%{hiera('bigtop::hadoop_head_node')}" -hadoop-hive::client::hbase_zookeeper_quorum: "%{hiera('hadoop-hbase::common_config::zookeeper_quorum')}" +hadoop_hive::client::hbase_zookeeper_quorum: "%{hiera('hadoop_hbase::common_config::zookeeper_quorum')}" # tez hadoop::common::tez_conf_dir: "/etc/tez/conf" @@ -168,4 +168,4 @@ hadoop::common::tez_jars: "/usr/lib/tez" # hadoop::common_mapred_app::mapreduce_framework_name: "yarn-tez" # to enable tez in hive, uncomment the lines below -# hadoop-hive::client::hive_execution_engine: "tez" +# hadoop_hive::client::hive_execution_engine: "tez" http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/hieradata/site.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml index 70c0529..e44cb50 100644 --- a/bigtop-deploy/puppet/hieradata/site.yaml +++ b/bigtop-deploy/puppet/hieradata/site.yaml @@ -15,7 +15,7 @@ hadoop::hadoop_storage_dirs: # - crunch # - flume # - giraph -# - ignite-hadoop +# - ignite_hadoop # - hbase # - hcat # - hive http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/manifests/cluster.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index 15a247b..14d078e 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -39,7 +39,7 @@ $roles_map = { worker => ["hbase-server"], client => ["hbase-client"], }, - ignite-hadoop => { + ignite_hadoop => { worker => ["ignite-server"], }, solrcloud => { @@ -150,14 +150,14 @@ class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_ "crunch", "giraph", "hadoop", - "hadoop-hbase", - "ignite-hadoop", - "hadoop-flume", - "hadoop-hive", - "hadoop-oozie", - "hadoop-pig", + "hadoop_hbase", + "ignite_hadoop", + "hadoop_flume", + "hadoop_hive", + "hadoop_oozie", + "hadoop_pig", "sqoop2", - "hadoop-zookeeper", + "hadoop_zookeeper", "hcatalog", "hue", "mahout", http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp deleted file mode 100644 index e6012cf..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class hadoop-flume { - - class deploy ($roles) { - if ("flume-agent" in $roles) { - include hadoop-flume::agent - } - } - - class agent($sources = [], $sinks = [], $channels = []) { - package { "flume-agent": - ensure => latest, - } - - file { - "/etc/flume/conf/flume.conf": - content => template('hadoop-flume/flume.conf'), - require => Package["flume-agent"], - } - - service { "flume-agent": - ensure => running, - require => [Package["flume-agent"], File["/etc/flume/conf/flume.conf"]], - subscribe => [Package["flume-agent"], File["/etc/flume/conf/flume.conf"]], - hasstatus => true, - hasrestart => true, - } - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-flume/templates/flume.conf ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-flume/templates/flume.conf b/bigtop-deploy/puppet/modules/hadoop-flume/templates/flume.conf deleted file mode 100644 index 7d1b1a2..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-flume/templates/flume.conf +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -agent.channels = <%= (@channels.collect { |k,v| k }).join(' ') %> -agent.sources = <%= (@sources.collect { |k,v| k }).join(' ') %> -agent.sinks = <%= (@sinks.collect { |k,v| k }).join(' ') %> - -<% @channels.each do |channel,props| -%> -<% props.each do |k,v| -%> -agent.channels.<%= channel %>.<%= k %> = <%= v %> -<% end -%> -<% end -%> - -<% @sources.each do |source,props| -%> -<% props.each do |k,v| -%> -agent.sources.<%= source %>.<%= k %> = <%= v %> -<% end -%> -<% end -%> - -<% @sinks.each do |sink,props| -%> -<% props.each do |k,v| -%> -agent.sinks.<%= sink %>.<%= k %> = <%= v %> -<% end -%> -<% end -%> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-flume/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-flume/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop-flume/tests/init.pp deleted file mode 100644 index 9694be6..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-flume/tests/init.pp +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -hadoop-flume::agent { "test-flume-agent": - channels => { mem => { type => "memory", - capacity => "1000", - transactionCapactiy => "100" - }, - file => { type => "file", - checkpointDir => "/var/lib/flume/checkpoint", - dataDirs => "/var/lib/flume/data", - } - }, - sources => { netcat => { type => "netcat", - bind => "0.0.0.0", - port => "1949", - channels => "mem" - }, - }, - sinks => { hdfs => { type => "hdfs", - "hdfs.path" => "/flume/test-flume-agent", - "hdfs.writeFormat" => "Text", - "hdfs.fileType" => "DataStream", - "hdfs.filePrefix" => "events-", - "hdfs.round" => "true", - "hdfs.roundValue" => "10", - "hdfs.roundUnit" => "minute", - "hdfs.serializer" => "org.apache.flume.serialization.BodyTextEventSerializer", - channel => "mem", - } - }, -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp deleted file mode 100644 index f61a23b..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp +++ /dev/null @@ -1,118 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class hadoop-hbase { - - class deploy ($roles) { - if ("hbase-server" in $roles) { - include hadoop-hbase::server - } - - if ("hbase-master" in $roles) { - include hadoop::init_hdfs - include hadoop-hbase::master - include hadoop-zookeeper::server - Class['Hadoop::Init_hdfs'] -> Class['Hadoop-hbase::Master'] - } - - if ("hbase-client" in $roles) { - include hadoop-hbase::client - } - } - - class client-package { - package { "hbase": - ensure => latest, - } - } - - class common_config ($rootdir, $zookeeper_quorum, $kerberos_realm = "", $heap_size="1024") { - include client-package - if ($kerberos_realm) { - require kerberos::client - kerberos::host_keytab { "hbase": - spnego => true, - require => Package["hbase"], - } - - file { "/etc/hbase/conf/jaas.conf": - content => template("hadoop-hbase/jaas.conf"), - require => Package["hbase"], - } - } - - file { "/etc/hbase/conf/hbase-site.xml": - content => template("hadoop-hbase/hbase-site.xml"), - require => Package["hbase"], - } - file { "/etc/hbase/conf/hbase-env.sh": - content => template("hadoop-hbase/hbase-env.sh"), - require => Package["hbase"], - } - } - - class client($thrift = false) { - include common_config - - if ($thrift) { - package { "hbase-thrift": - ensure => latest, - } - - service { "hbase-thrift": - ensure => running, - require => Package["hbase-thrift"], - subscribe => File["/etc/hbase/conf/hbase-site.xml", "/etc/hbase/conf/hbase-env.sh"], - hasrestart => true, - hasstatus => true, - } - Kerberos::Host_keytab <| title == "hbase" |> -> Service["hbase-thrift"] - } - } - - class server { - include common_config - - package { "hbase-regionserver": - ensure => latest, - } - - service { "hbase-regionserver": - ensure => running, - require => Package["hbase-regionserver"], - subscribe => File["/etc/hbase/conf/hbase-site.xml", "/etc/hbase/conf/hbase-env.sh"], - hasrestart => true, - hasstatus => true, - } - Kerberos::Host_keytab <| title == "hbase" |> -> Service["hbase-regionserver"] - } - - class master { - include common_config - - package { "hbase-master": - ensure => latest, - } - - service { "hbase-master": - ensure => running, - require => Package["hbase-master"], - subscribe => File["/etc/hbase/conf/hbase-site.xml", "/etc/hbase/conf/hbase-env.sh"], - hasrestart => true, - hasstatus => true, - } - Kerberos::Host_keytab <| title == "hbase" |> -> Service["hbase-master"] - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-env.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-env.sh b/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-env.sh deleted file mode 100644 index 2f96581..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-env.sh +++ /dev/null @@ -1,82 +0,0 @@ -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Set environment variables here. - -# The java implementation to use. Java 1.6 required. -# export JAVA_HOME=/usr/java/jdk1.6.0/ - -# Extra Java CLASSPATH elements. Optional. -export HBASE_CLASSPATH=/etc/hadoop/conf - -# The maximum amount of heap to use, in MB. Default is 1000. -export HBASE_HEAPSIZE=<%= @heap_size %> - -# Extra Java runtime options. -# Below are what we set by default. May only work with SUN JVM. -# For more on why as well as other possible settings, -# see http://wiki.apache.org/hadoop/PerformanceTuning -export HBASE_OPTS="$HBASE_OPTS -ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode" - -<% if @kerberos_realm != "" -%> -# Secure Zookeeper settings -export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config=/etc/hbase/conf/jaas.conf" -export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config=/etc/hbase/conf/jaas.conf" -<% end -%> - -# Uncomment below to enable java garbage collection logging. -# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" - -# Uncomment and adjust to enable JMX exporting -# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. -# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# -# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" -# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" -# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" -# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" -# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" - -# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. -# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers - -# Extra ssh options. Empty by default. -# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" - -# Where log files are stored. $HBASE_HOME/logs by default. -# export HBASE_LOG_DIR=${HBASE_HOME}/logs - -# A string representing this instance of hbase. $USER by default. -# export HBASE_IDENT_STRING=$USER - -# The scheduling priority for daemon processes. See 'man nice'. -# export HBASE_NICENESS=10 - -# The directory where pid files are stored. /tmp by default. -# export HBASE_PID_DIR=/var/hadoop/pids - -# Seconds to sleep between slave commands. Unset by default. This -# can be useful in large clusters, where, e.g., slave rsyncs can -# otherwise arrive faster than the master can service them. -# export HBASE_SLAVE_SLEEP=0.1 - -# Tell HBase whether it should manage it's own instance of Zookeeper or not. -# export HBASE_MANAGES_ZK=true http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-site.xml b/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-site.xml deleted file mode 100644 index 849676c..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-site.xml +++ /dev/null @@ -1,70 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Copyright 2010 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration> - <property> - <name>hbase.cluster.distributed</name> - <value>true</value> - </property> - - <property> - <name>hbase.zookeeper.quorum</name> - <value><%= @zookeeper_quorum %></value> - </property> - - <property> - <name>hbase.rootdir</name> - <value><%= @rootdir %></value> - </property> - - <property> - <name>dfs.support.append</name> - <value>true</value> - </property> - -<% if @hdfs_shortcut_reader_user %> - <property> - <name>dfs.client.read.shortcircuit</name> - <value>true</value> - </property> -<% end %> - -<% if @kerberos_realm != "" %> - <property> - <name>hbase.regionserver.kerberos.principal</name> - <value>hbase/_HOST@<%= @kerberos_realm %></value> - </property> - <property> - <name>hbase.regionserver.keytab.file</name> - <value>/etc/hbase.keytab</value> - </property> - <property> - <name>hbase.master.kerberos.principal</name> - <value>hbase/_HOST@<%= @kerberos_realm %></value> - </property> - <property> - <name>hbase.master.keytab.file</name> - <value>/etc/hbase.keytab</value> - </property> -<% end %> -</configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-hbase/templates/jaas.conf ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-hbase/templates/jaas.conf b/bigtop-deploy/puppet/modules/hadoop-hbase/templates/jaas.conf deleted file mode 100644 index 5b6b2a5..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-hbase/templates/jaas.conf +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -Client { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - useTicketCache=false - keyTab="/etc/hbase.keytab" - principal="hbase/<%= @fqdn %>@<%= @kerberos_realm %>"; -}; http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp deleted file mode 100644 index c6fc7c0..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -hadoop-hbase::client { "hbase thrift client": - thrift => true, - kerberos_realm => "KRB.YOU.ORG", -} -hadoop-hbase::master { "test-hbase-master": - rootdir => "hdfs://localhost:17020/hbase", - zookeeper_quorum => "localhost", - kerberos_realm => "KRB.YOU.ORG", -} -hadoop-hbase::server { "test-hbase-server": - rootdir => "hdfs://localhost:17020/hbase", - zookeeper_quorum => "localhost", - kerberos_realm => "KRB.YOU.ORG", -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp deleted file mode 100644 index b0eee0a..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class hadoop-hive { - - class deploy ($roles) { - if ("hive-client" in $roles) { - include hadoop-hive::client - } - } - - class client($hbase_master = "", - $hbase_zookeeper_quorum = "", - $hive_execution_engine = "mr") { - - package { "hive": - ensure => latest, - } - - file { "/etc/hive/conf/hive-site.xml": - content => template('hadoop-hive/hive-site.xml'), - require => Package["hive"], - } - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml b/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml deleted file mode 100644 index 2aca237..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml +++ /dev/null @@ -1,68 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - -<!-- Licensed to the Apache Software Foundation (ASF) under one or more --> -<!-- contributor license agreements. See the NOTICE file distributed with --> -<!-- this work for additional information regarding copyright ownership. --> -<!-- The ASF licenses this file to You under the Apache License, Version 2.0 --> -<!-- (the "License"); you may not use this file except in compliance with --> -<!-- the License. You may obtain a copy of the License at --> -<!-- --> -<!-- http://www.apache.org/licenses/LICENSE-2.0 --> -<!-- --> -<!-- Unless required by applicable law or agreed to in writing, software --> -<!-- distributed under the License is distributed on an "AS IS" BASIS, --> -<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --> -<!-- See the License for the specific language governing permissions and --> -<!-- limitations under the License. --> - -<configuration> - -<!-- Hive Configuration can either be stored in this file or in the hadoop configuration files --> -<!-- that are implied by Hadoop setup variables. --> -<!-- Aside from Hadoop setup variables - this file is provided as a convenience so that Hive --> -<!-- users do not have to edit hadoop configuration files (that may be managed as a centralized --> -<!-- resource). --> - -<!-- Hive Execution Parameters --> - -<% if @hbase_master != "" %> -<property> - <name>hbase.master</name> - <value><%= @hbase_master %></value> - <description>http://wiki.apache.org/hadoop/Hive/HBaseIntegration</description> -</property> -<% end %> - -<% if @hbase_zookeeper_quorum != "" %> -<property> - <name>hbase.zookeeper.quorum</name> - <value><%= @hbase_zookeeper_quorum %></value> - <description>http://wiki.apache.org/hadoop/Hive/HBaseIntegration</description> -</property> -<% end %> - -<property> - <name>hive.execution.engine</name> - <value><%= @hive_execution_engine %></value> -</property> - -<property> - <name>javax.jdo.option.ConnectionURL</name> - <value>jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true</value> - <description>JDBC connect string for a JDBC metastore</description> -</property> - -<property> - <name>javax.jdo.option.ConnectionDriverName</name> - <value>org.apache.derby.jdbc.EmbeddedDriver</value> - <description>Driver class name for a JDBC metastore</description> -</property> - -<property> - <name>hive.hwi.war.file</name> - <value>/usr/lib/hive/lib/hive-hwi.war</value> - <description>This is the WAR file with the jsp content for Hive Web Interface</description> -</property> - -</configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-hive/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-hive/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop-hive/tests/init.pp deleted file mode 100644 index 44c62c8..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-hive/tests/init.pp +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -hadoop-hive::client { "test-hive": } http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp deleted file mode 100644 index 0829360..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp +++ /dev/null @@ -1,74 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class hadoop-oozie { - - class deploy ($roles) { - if ("oozie-client" in $roles) { - include hadoop-oozie::client - } - - if ("oozie-server" in $roles) { - include hadoop::init_hdfs - include hadoop-oozie::server - Class['Hadoop::Init_hdfs'] -> Class['Hadoop-oozie::Server'] - if ("mapred-app" in $roles) { - Class['Hadoop::Mapred-app'] -> Class['Hadoop-oozie::Server'] - } - } - } - - class client { - package { "oozie-client": - ensure => latest, - } - } - - class server($kerberos_realm = "") { - if ($kerberos_realm) { - require kerberos::client - kerberos::host_keytab { "oozie": - spnego => true, - require => Package["oozie"], - } - } - - package { "oozie": - ensure => latest, - } - - file { "/etc/oozie/conf/oozie-site.xml": - content => template("hadoop-oozie/oozie-site.xml"), - require => Package["oozie"], - } - - exec { "Oozie DB init": - command => "/etc/init.d/oozie init", - cwd => "/var/lib/oozie", - creates => "/var/lib/oozie/derby.log", - require => Package["oozie"], - unless => "/etc/init.d/oozie status", - } - - service { "oozie": - ensure => running, - require => [ Package["oozie"], Exec["Oozie DB init"] ], - hasrestart => true, - hasstatus => true, - } - Kerberos::Host_keytab <| title == "oozie" |> -> Service["oozie"] - - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml b/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml deleted file mode 100644 index 93d5eaf..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml +++ /dev/null @@ -1,369 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<configuration> - - <!-- - Refer to the oozie-default.xml file for the complete list of - Oozie configuration properties and their default values. - --> - - <property> - <name>oozie.service.ActionService.executor.ext.classes</name> - <value> - org.apache.oozie.action.email.EmailActionExecutor, - org.apache.oozie.action.hadoop.HiveActionExecutor, - org.apache.oozie.action.hadoop.ShellActionExecutor, - org.apache.oozie.action.hadoop.SqoopActionExecutor, - org.apache.oozie.action.hadoop.DistcpActionExecutor - </value> - </property> - - <property> - <name>oozie.service.SchemaService.wf.ext.schemas</name> - <value>shell-action-0.1.xsd,shell-action-0.2.xsd,email-action-0.1.xsd,hive-action-0.2.xsd,hive-action-0.3.xsd,sqoop-action-0.2.xsd,sqoop-action-0.3.xsd,ssh-action-0.1.xsd,distcp-action-0.1.xsd</value> - </property> - - <property> - <name>oozie.system.id</name> - <value>oozie-${user.name}</value> - <description> - The Oozie system ID. - </description> - </property> - - <property> - <name>oozie.systemmode</name> - <value>NORMAL</value> - <description> - System mode for Oozie at startup. - </description> - </property> - - <property> - <name>oozie.service.AuthorizationService.security.enabled</name> - <value>false</value> - <description> - Specifies whether security (user name/admin role) is enabled or not. - If disabled any user can manage Oozie system and manage any job. - </description> - </property> - - <property> - <name>oozie.service.PurgeService.older.than</name> - <value>30</value> - <description> - Jobs older than this value, in days, will be purged by the PurgeService. - </description> - </property> - - <property> - <name>oozie.service.PurgeService.purge.interval</name> - <value>3600</value> - <description> - Interval at which the purge service will run, in seconds. - </description> - </property> - - <property> - <name>oozie.service.CallableQueueService.queue.size</name> - <value>10000</value> - <description>Max callable queue size</description> - </property> - - <property> - <name>oozie.service.CallableQueueService.threads</name> - <value>10</value> - <description>Number of threads used for executing callables</description> - </property> - - <property> - <name>oozie.service.CallableQueueService.callable.concurrency</name> - <value>3</value> - <description> - Maximum concurrency for a given callable type. - Each command is a callable type (submit, start, run, signal, job, jobs, suspend,resume, etc). - Each action type is a callable type (Map-Reduce, Pig, SSH, FS, sub-workflow, etc). - All commands that use action executors (action-start, action-end, action-kill and action-check) use - the action type as the callable type. - </description> - </property> - - <property> - <name>oozie.service.coord.normal.default.timeout - </name> - <value>120</value> - <description>Default timeout for a coordinator action input check (in minutes) for normal job. - -1 means infinite timeout</description> - </property> - - <property> - <name>oozie.db.schema.name</name> - <value>oozie</value> - <description> - Oozie DataBase Name - </description> - </property> - - <property> - <name>oozie.service.JPAService.create.db.schema</name> - <value>false</value> - <description> - Creates Oozie DB. - - If set to true, it creates the DB schema if it does not exist. If the DB schema exists is a NOP. - If set to false, it does not create the DB schema. If the DB schema does not exist it fails start up. - </description> - </property> - - <property> - <name>oozie.service.JPAService.jdbc.driver</name> - <value>org.apache.derby.jdbc.EmbeddedDriver</value> - <description> - JDBC driver class. - </description> - </property> - - <property> - <name>oozie.service.JPAService.jdbc.url</name> - <value>jdbc:derby:${oozie.data.dir}/${oozie.db.schema.name}-db;create=true</value> - <description> - JDBC URL. - </description> - </property> - - <property> - <name>oozie.service.JPAService.jdbc.username</name> - <value>sa</value> - <description> - DB user name. - </description> - </property> - - <property> - <name>oozie.service.JPAService.jdbc.password</name> - <value> </value> - <description> - DB user password. - - IMPORTANT: if password is emtpy leave a 1 space string, the service trims the value, - if empty Configuration assumes it is NULL. - </description> - </property> - - <property> - <name>oozie.service.JPAService.pool.max.active.conn</name> - <value>10</value> - <description> - Max number of connections. - </description> - </property> - - <property> - <name>oozie.service.HadoopAccessorService.kerberos.enabled</name> - <value><%= if (@kerberos_realm != "") ; "true" else "false" end %></value> - <description> - Indicates if Oozie is configured to use Kerberos. - </description> - </property> - - <property> - <name>local.realm</name> - <value><%= @kerberos_realm %></value> - <description> - Kerberos Realm used by Oozie and Hadoop. Using 'local.realm' to be aligned with Hadoop configuration - </description> - </property> - - <property> - <name>oozie.service.HadoopAccessorService.keytab.file</name> - <value>/etc/oozie.keytab</value> - <description> - Location of the Oozie user keytab file. - </description> - </property> - - <property> - <name>oozie.service.HadoopAccessorService.kerberos.principal</name> - <value>${user.name}/<%= @fqdn %>@${local.realm}</value> - <description> - Kerberos principal for Oozie service. - </description> - </property> - - <property> - <name>oozie.service.HadoopAccessorService.jobTracker.whitelist</name> - <value> </value> - <description> - Whitelisted job tracker for Oozie service. - </description> - </property> - - <property> - <name>oozie.service.HadoopAccessorService.nameNode.whitelist</name> - <value> </value> - <description> - Whitelisted job tracker for Oozie service. - </description> - </property> - - <property> - <name>oozie.service.HadoopAccessorService.hadoop.configurations</name> - <value>*=/etc/hadoop/conf</value> - <description> - Comma separated AUTHORITY=HADOOP_CONF_DIR, where AUTHORITY is the HOST:PORT of - the Hadoop service (JobTracker, HDFS). The wildcard '*' configuration is - used when there is no exact match for an authority. The HADOOP_CONF_DIR contains - the relevant Hadoop *-site.xml files. If the path is relative is looked within - the Oozie configuration directory; though the path can be absolute (i.e. to point - to Hadoop client conf/ directories in the local filesystem. - </description> - </property> - - <property> - <name>oozie.service.WorkflowAppService.system.libpath</name> - <value>/user/${user.name}/share/lib</value> - <description> - System library path to use for workflow applications. - This path is added to workflow application if their job properties sets - the property 'oozie.use.system.libpath' to true. - </description> - </property> - - <property> - <name>use.system.libpath.for.mapreduce.and.pig.jobs</name> - <value>false</value> - <description> - If set to true, submissions of MapReduce and Pig jobs will include - automatically the system library path, thus not requiring users to - specify where the Pig JAR files are. Instead, the ones from the system - library path are used. - </description> - </property> - - <property> - <name>oozie.authentication.type</name> - <value><%= @kerberos_realm.empty? ? "simple" : "kerberos" %></value> - <description> - Defines authentication used for Oozie HTTP endpoint. - Supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME# - </description> - </property> - - <property> - <name>oozie.authentication.token.validity</name> - <value>36000</value> - <description> - Indicates how long (in seconds) an authentication token is valid before it has - to be renewed. - </description> - </property> - - <property> - <name>oozie.authentication.signature.secret</name> - <value>oozie</value> - <description> - The signature secret for signing the authentication tokens. - If not set a random secret is generated at startup time. - In order to authentiation to work correctly across multiple hosts - the secret must be the same across al the hosts. - </description> - </property> - - <property> - <name>oozie.authentication.cookie.domain</name> - <value></value> - <description> - The domain to use for the HTTP cookie that stores the authentication token. - In order to authentiation to work correctly across multiple hosts - the domain must be correctly set. - </description> - </property> - - <property> - <name>oozie.authentication.simple.anonymous.allowed</name> - <value>true</value> - <description> - Indicates if anonymous requests are allowed. - This setting is meaningful only when using 'simple' authentication. - </description> - </property> - - <property> - <name>oozie.authentication.kerberos.principal</name> - <value>HTTP/<%= @fqdn %>@${local.realm}</value> - <description> - Indicates the Kerberos principal to be used for HTTP endpoint. - The principal MUST start with 'HTTP/' as per Kerberos HTTP SPNEGO specification. - </description> - </property> - - <property> - <name>oozie.authentication.kerberos.keytab</name> - <value>${oozie.service.HadoopAccessorService.keytab.file}</value> - <description> - Location of the keytab file with the credentials for the principal. - Referring to the same keytab file Oozie uses for its Kerberos credentials for Hadoop. - </description> - </property> - - <property> - <name>oozie.authentication.kerberos.name.rules</name> - <value>DEFAULT</value> - <description> - The kerberos names rules is to resolve kerberos principal names, refer to Hadoop's - KerberosName for more details. - </description> - </property> - - <!-- Proxyuser Configuration --> - - <property> - <name>oozie.service.ProxyUserService.proxyuser.hue.hosts</name> - <value>*</value> - <description> - List of hosts the '#USER#' user is allowed to perform 'doAs' - operations. - - The '#USER#' must be replaced with the username o the user who is - allowed to perform 'doAs' operations. - - The value can be the '*' wildcard or a list of hostnames. - - For multiple users copy this property and replace the user name - in the property name. - </description> - </property> - - <property> - <name>oozie.service.ProxyUserService.proxyuser.hue.groups</name> - <value>*</value> - <description> - List of groups the '#USER#' user is allowed to impersonate users - from to perform 'doAs' operations. - - The '#USER#' must be replaced with the username o the user who is - allowed to perform 'doAs' operations. - - The value can be the '*' wildcard or a list of groups. - - For multiple users copy this property and replace the user name - in the property name. - </description> - </property> -</configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-oozie/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-oozie/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop-oozie/tests/init.pp deleted file mode 100644 index 9600742..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-oozie/tests/init.pp +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -hadoop-oozie::server { "test-oozie-server": } -hadoop-oozie::client { "test-oozie": } http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp deleted file mode 100644 index 3ffaa70..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class hadoop-pig { - - class deploy ($roles) { - if ("pig-client" in $roles) { - include hadoop-pig::client - } - } - - class client { - package { "pig": - ensure => latest, - require => Package["hadoop"], - } - - file { "/etc/pig/conf/pig.properties": - content => template('hadoop-pig/pig.properties'), - require => Package["pig"], - owner => "root", /* FIXME: I'm really no sure about these -- we might end */ - mode => "755", /* up deploying/testing a different thing compared */ - /* to a straight rpm/deb deployment */ - } - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-pig/templates/pig.properties ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-pig/templates/pig.properties b/bigtop-deploy/puppet/modules/hadoop-pig/templates/pig.properties deleted file mode 100644 index 1e1c297..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-pig/templates/pig.properties +++ /dev/null @@ -1,70 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Pig configuration file. All values can be overwritten by command line arguments. -# see bin/pig -help - -# log4jconf log4j configuration file -# log4jconf=./conf/log4j.properties - -# brief logging (no timestamps) -brief=false - -# clustername, name of the hadoop jobtracker. If no port is defined port 50020 will be used. -#cluster - -#debug level, INFO is default -debug=INFO - -# a file that contains pig script -#file= - -# load jarfile, colon separated -#jar= - -#verbose print all log messages to screen (default to print only INFO and above to screen) -verbose=false - -#exectype local|mapreduce, mapreduce is default -#exectype=mapreduce -# hod realted properties -#ssh.gateway -#hod.expect.root -#hod.expect.uselatest -#hod.command -#hod.config.dir -#hod.param - - -#Do not spill temp files smaller than this size (bytes) -pig.spill.size.threshold=5000000 -#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) -#This should help reduce the number of files being spilled. -pig.spill.gc.activation.size=40000000 - - -###################### -# Everything below this line is Yahoo specific. Note that I've made -# (almost) no changes to the lines above to make merging in from Apache -# easier. Any values I don't want from above I override below. -# -# This file is configured for use with HOD on the production clusters. If you -# want to run pig with a static cluster you will need to remove everything -# below this line and set the cluster value (above) to the -# hostname and port of your job tracker. - -exectype=mapreduce -log.file= http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-pig/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-pig/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop-pig/tests/init.pp deleted file mode 100644 index cdd9780..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-pig/tests/init.pp +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -include hadoop-pig -hadoop-pig::client { "test-pig": } http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-zookeeper/files/java.env ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-zookeeper/files/java.env b/bigtop-deploy/puppet/modules/hadoop-zookeeper/files/java.env deleted file mode 100644 index 6aca488..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-zookeeper/files/java.env +++ /dev/null @@ -1,15 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -export SERVER_JVMFLAGS="-Djava.security.auth.login.config=/etc/zookeeper/conf/jaas.conf" http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp deleted file mode 100644 index a44834c..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp +++ /dev/null @@ -1,95 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class hadoop-zookeeper { - - class deploy ($roles) { - if ("zookeeper-client" in $roles) { - include hadoop-zookeeper::client - } - - if ("zookeeper-server" in $roles) { - include hadoop-zookeeper::server - } - } - - class client { - package { "zookeeper": - ensure => latest, - require => Package["jdk"], - } - } - - class server($myid, - $port = "2181", - $datadir = "/var/lib/zookeeper", - $ensemble = ["localhost:2888:3888"], - $kerberos_realm = "") - { - package { "zookeeper-server": - ensure => latest, - require => Package["jdk"], - } - - service { "zookeeper-server": - ensure => running, - require => [ Package["zookeeper-server"], - Exec["zookeeper-server-initialize"] ], - subscribe => [ File["/etc/zookeeper/conf/zoo.cfg"], - File["/var/lib/zookeeper/myid"] ], - hasrestart => true, - hasstatus => true, - } - - file { "/etc/zookeeper/conf/zoo.cfg": - content => template("hadoop-zookeeper/zoo.cfg"), - require => Package["zookeeper-server"], - } - - file { "/var/lib/zookeeper/myid": - content => inline_template("<%= @myid %>"), - require => Package["zookeeper-server"], - } - - exec { "zookeeper-server-initialize": - command => "/usr/bin/zookeeper-server-initialize", - user => "zookeeper", - creates => "/var/lib/zookeeper/version-2", - require => Package["zookeeper-server"], - } - - if ($kerberos_realm) { - require kerberos::client - - kerberos::host_keytab { "zookeeper": - spnego => true, - notify => Service["zookeeper-server"], - require => Package["zookeeper-server"], - } - - file { "/etc/zookeeper/conf/java.env": - source => "puppet:///modules/hadoop-zookeeper/java.env", - require => Package["zookeeper-server"], - notify => Service["zookeeper-server"], - } - - file { "/etc/zookeeper/conf/jaas.conf": - content => template("hadoop-zookeeper/jaas.conf"), - require => Package["zookeeper-server"], - notify => Service["zookeeper-server"], - } - } - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/jaas.conf ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/jaas.conf b/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/jaas.conf deleted file mode 100644 index 61b2eac..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/jaas.conf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -Server { - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - keyTab="/etc/zookeeper.keytab" - storeKey=true - useTicketCache=false - principal="zookeeper/<%= @fqdn %>@<%= @kerberos_realm %>"; -}; http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/zoo.cfg ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/zoo.cfg b/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/zoo.cfg deleted file mode 100644 index 426fe2a..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/zoo.cfg +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -maxClientCnxns=50 -# The number of milliseconds of each tick -tickTime=2000 -# The number of ticks that the initial -# synchronization phase can take -initLimit=10 -# The number of ticks that can pass between -# sending a request and getting an acknowledgement -syncLimit=5 -# the directory where the snapshot is stored. -dataDir=<%= @datadir %> -# the port at which the clients will connect -clientPort=<%= @port %> -<% @ensemble.each_with_index do |server,idx| %> -server.<%= idx %>=<%= server %> -<% end %> - -<% if @kerberos_realm != "" -%> -authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider -jaasLoginRenew=3600000 - -# HBase needs these configs so that different hbase daemons -# (master, regionservers), which run on different hosts, can -# read from and write to znodes that others create -kerberos.removeHostFromPrincipal=true -kerberos.removeRealmFromPrincipal=true -<% end -%> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop-zookeeper/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop-zookeeper/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop-zookeeper/tests/init.pp deleted file mode 100644 index 55a3160..0000000 --- a/bigtop-deploy/puppet/modules/hadoop-zookeeper/tests/init.pp +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -hadoop-zookeeper::client { "zoo visitor": } -hadoop-zookeeper::server { "test-oozie-server": - myid => "0", - ensemble => ["foo:2888:3888", "bar:2888:3888", "baz:2888:3888"], -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_flume/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_flume/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop_flume/manifests/init.pp new file mode 100644 index 0000000..32f80c9 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_flume/manifests/init.pp @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class hadoop_flume { + + class deploy ($roles) { + if ("flume-agent" in $roles) { + include hadoop_flume::agent + } + } + + class agent($sources = [], $sinks = [], $channels = []) { + package { "flume-agent": + ensure => latest, + } + + file { + "/etc/flume/conf/flume.conf": + content => template('hadoop_flume/flume.conf'), + require => Package["flume-agent"], + } + + service { "flume-agent": + ensure => running, + require => [Package["flume-agent"], File["/etc/flume/conf/flume.conf"]], + subscribe => [Package["flume-agent"], File["/etc/flume/conf/flume.conf"]], + hasstatus => true, + hasrestart => true, + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_flume/templates/flume.conf ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_flume/templates/flume.conf b/bigtop-deploy/puppet/modules/hadoop_flume/templates/flume.conf new file mode 100644 index 0000000..7d1b1a2 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_flume/templates/flume.conf @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +agent.channels = <%= (@channels.collect { |k,v| k }).join(' ') %> +agent.sources = <%= (@sources.collect { |k,v| k }).join(' ') %> +agent.sinks = <%= (@sinks.collect { |k,v| k }).join(' ') %> + +<% @channels.each do |channel,props| -%> +<% props.each do |k,v| -%> +agent.channels.<%= channel %>.<%= k %> = <%= v %> +<% end -%> +<% end -%> + +<% @sources.each do |source,props| -%> +<% props.each do |k,v| -%> +agent.sources.<%= source %>.<%= k %> = <%= v %> +<% end -%> +<% end -%> + +<% @sinks.each do |sink,props| -%> +<% props.each do |k,v| -%> +agent.sinks.<%= sink %>.<%= k %> = <%= v %> +<% end -%> +<% end -%> http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_flume/tests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_flume/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop_flume/tests/init.pp new file mode 100644 index 0000000..4fbd699 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_flume/tests/init.pp @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +hadoop_flume::agent { "test-flume-agent": + channels => { mem => { type => "memory", + capacity => "1000", + transactionCapactiy => "100" + }, + file => { type => "file", + checkpointDir => "/var/lib/flume/checkpoint", + dataDirs => "/var/lib/flume/data", + } + }, + sources => { netcat => { type => "netcat", + bind => "0.0.0.0", + port => "1949", + channels => "mem" + }, + }, + sinks => { hdfs => { type => "hdfs", + "hdfs.path" => "/flume/test-flume-agent", + "hdfs.writeFormat" => "Text", + "hdfs.fileType" => "DataStream", + "hdfs.filePrefix" => "events-", + "hdfs.round" => "true", + "hdfs.roundValue" => "10", + "hdfs.roundUnit" => "minute", + "hdfs.serializer" => "org.apache.flume.serialization.BodyTextEventSerializer", + channel => "mem", + } + }, +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_hbase/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_hbase/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop_hbase/manifests/init.pp new file mode 100644 index 0000000..1527260 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_hbase/manifests/init.pp @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class hadoop_hbase { + + class deploy ($roles) { + if ("hbase-server" in $roles) { + include hadoop_hbase::server + } + + if ("hbase-master" in $roles) { + include hadoop::init_hdfs + include hadoop_hbase::master + include hadoop_zookeeper::server + Class['Hadoop::Init_hdfs'] -> Class['Hadoop_hbase::Master'] + } + + if ("hbase-client" in $roles) { + include hadoop_hbase::client + } + } + + class client-package { + package { "hbase": + ensure => latest, + } + } + + class common_config ($rootdir, $zookeeper_quorum, $kerberos_realm = "", $heap_size="1024") { + include client-package + if ($kerberos_realm) { + require kerberos::client + kerberos::host_keytab { "hbase": + spnego => true, + require => Package["hbase"], + } + + file { "/etc/hbase/conf/jaas.conf": + content => template("hadoop_hbase/jaas.conf"), + require => Package["hbase"], + } + } + + file { "/etc/hbase/conf/hbase-site.xml": + content => template("hadoop_hbase/hbase-site.xml"), + require => Package["hbase"], + } + file { "/etc/hbase/conf/hbase-env.sh": + content => template("hadoop_hbase/hbase-env.sh"), + require => Package["hbase"], + } + } + + class client($thrift = false) { + include common_config + + if ($thrift) { + package { "hbase-thrift": + ensure => latest, + } + + service { "hbase-thrift": + ensure => running, + require => Package["hbase-thrift"], + subscribe => File["/etc/hbase/conf/hbase-site.xml", "/etc/hbase/conf/hbase-env.sh"], + hasrestart => true, + hasstatus => true, + } + Kerberos::Host_keytab <| title == "hbase" |> -> Service["hbase-thrift"] + } + } + + class server { + include common_config + + package { "hbase-regionserver": + ensure => latest, + } + + service { "hbase-regionserver": + ensure => running, + require => Package["hbase-regionserver"], + subscribe => File["/etc/hbase/conf/hbase-site.xml", "/etc/hbase/conf/hbase-env.sh"], + hasrestart => true, + hasstatus => true, + } + Kerberos::Host_keytab <| title == "hbase" |> -> Service["hbase-regionserver"] + } + + class master { + include common_config + + package { "hbase-master": + ensure => latest, + } + + service { "hbase-master": + ensure => running, + require => Package["hbase-master"], + subscribe => File["/etc/hbase/conf/hbase-site.xml", "/etc/hbase/conf/hbase-env.sh"], + hasrestart => true, + hasstatus => true, + } + Kerberos::Host_keytab <| title == "hbase" |> -> Service["hbase-master"] + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_hbase/templates/hbase-env.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_hbase/templates/hbase-env.sh b/bigtop-deploy/puppet/modules/hadoop_hbase/templates/hbase-env.sh new file mode 100644 index 0000000..2f96581 --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_hbase/templates/hbase-env.sh @@ -0,0 +1,82 @@ +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +# export JAVA_HOME=/usr/java/jdk1.6.0/ + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=/etc/hadoop/conf + +# The maximum amount of heap to use, in MB. Default is 1000. +export HBASE_HEAPSIZE=<%= @heap_size %> + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export HBASE_OPTS="$HBASE_OPTS -ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode" + +<% if @kerberos_realm != "" -%> +# Secure Zookeeper settings +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config=/etc/hbase/conf/jaas.conf" +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config=/etc/hbase/conf/jaas.conf" +<% end -%> + +# Uncomment below to enable java garbage collection logging. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +# export HBASE_LOG_DIR=${HBASE_HOME}/logs + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +# export HBASE_PID_DIR=/var/hadoop/pids + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +# export HBASE_MANAGES_ZK=true http://git-wip-us.apache.org/repos/asf/bigtop/blob/eab70662/bigtop-deploy/puppet/modules/hadoop_hbase/templates/hbase-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_hbase/templates/hbase-site.xml b/bigtop-deploy/puppet/modules/hadoop_hbase/templates/hbase-site.xml new file mode 100644 index 0000000..849676c --- /dev/null +++ b/bigtop-deploy/puppet/modules/hadoop_hbase/templates/hbase-site.xml @@ -0,0 +1,70 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>hbase.cluster.distributed</name> + <value>true</value> + </property> + + <property> + <name>hbase.zookeeper.quorum</name> + <value><%= @zookeeper_quorum %></value> + </property> + + <property> + <name>hbase.rootdir</name> + <value><%= @rootdir %></value> + </property> + + <property> + <name>dfs.support.append</name> + <value>true</value> + </property> + +<% if @hdfs_shortcut_reader_user %> + <property> + <name>dfs.client.read.shortcircuit</name> + <value>true</value> + </property> +<% end %> + +<% if @kerberos_realm != "" %> + <property> + <name>hbase.regionserver.kerberos.principal</name> + <value>hbase/_HOST@<%= @kerberos_realm %></value> + </property> + <property> + <name>hbase.regionserver.keytab.file</name> + <value>/etc/hbase.keytab</value> + </property> + <property> + <name>hbase.master.kerberos.principal</name> + <value>hbase/_HOST@<%= @kerberos_realm %></value> + </property> + <property> + <name>hbase.master.keytab.file</name> + <value>/etc/hbase.keytab</value> + </property> +<% end %> +</configuration>
