Repository: bigtop Updated Branches: refs/heads/master 014ee22eb -> 0c82c31f7
BIGTOP-2578. Puppet needs to be updated for Hive Server2 Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/0c82c31f Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/0c82c31f Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/0c82c31f Branch: refs/heads/master Commit: 0c82c31f77fee6f749a1295e20fd3cd8d20c2c4a Parents: 35fdeb8 Author: Roman Shaposhnik <[email protected]> Authored: Mon Nov 7 13:54:17 2016 -0800 Committer: Roman Shaposhnik <[email protected]> Committed: Mon Nov 7 23:32:39 2016 -0800 ---------------------------------------------------------------------- .../puppet/hieradata/bigtop/cluster.yaml | 5 +- bigtop-deploy/puppet/manifests/cluster.pp | 1 + .../puppet/modules/hadoop/manifests/init.pp | 7 +-- .../modules/hadoop_hive/manifests/init.pp | 55 ++++++++++++++++++-- .../modules/hadoop_hive/templates/hive-site.xml | 34 ++++++++++++ 5 files changed, 93 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c82c31f/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml index 49904c6..89ac967 100644 --- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml +++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml @@ -186,7 +186,8 @@ hue::server::kerberos_realm: "%{hiera('kerberos::site::realm')}" giraph::client::zookeeper_quorum: "%{hiera('bigtop::hadoop_head_node')}" -hadoop_hive::client::hbase_zookeeper_quorum: "%{hiera('hadoop_hbase::common_config::zookeeper_quorum')}" +hadoop_hive::common_config::hbase_zookeeper_quorum: "%{hiera('hadoop_hbase::common_config::zookeeper_quorum')}" +hadoop_hive::common_config::kerberos_realm: "%{hiera('kerberos::site::realm')}" # tez hadoop::common::tez_conf_dir: "/etc/tez/conf" @@ -197,7 +198,7 @@ hadoop::common::tez_jars: "/usr/lib/tez" # hadoop::common_mapred_app::mapreduce_framework_name: "yarn-tez" # to enable tez in hive, uncomment the lines below -# hadoop_hive::client::hive_execution_engine: "tez" +# hadoop_hive::common_config::hive_execution_engine: "tez" #kafka kafka::server::port: "9092" http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c82c31f/bigtop-deploy/puppet/manifests/cluster.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp index 7144f02..7d31ef9 100644 --- a/bigtop-deploy/puppet/manifests/cluster.pp +++ b/bigtop-deploy/puppet/manifests/cluster.pp @@ -97,6 +97,7 @@ $roles_map = { client => ["pig-client"], }, hive => { + master => ["hive-server2"], client => ["hive-client"], }, tez => { http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c82c31f/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp index 3d357bc..a8a8152 100644 --- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp @@ -19,9 +19,10 @@ class hadoop ($hadoop_security_authentication = "simple", # Set from facter if available $hadoop_storage_dirs = split($::hadoop_storage_dirs, ";"), $proxyusers = { - oozie => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,httpfs,hue,users', hosts => "*" }, - hue => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,httpfs,hue,users', hosts => "*" }, - httpfs => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,httpfs,hue,users', hosts => "*" } }, + oozie => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,hive,httpfs,hue,users', hosts => "*" }, + hive => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,hive,httpfs,hue,users', hosts => "*" }, + hue => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,hive,httpfs,hue,users', hosts => "*" }, + httpfs => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,hive,httpfs,hue,users', hosts => "*" } }, $generate_secrets = false, ) { http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c82c31f/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp index 86131ff..4b10a25 100644 --- a/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop_hive/manifests/init.pp @@ -19,19 +19,66 @@ class hadoop_hive { if ("hive-client" in $roles) { include hadoop_hive::client } - } - class client($hbase_master = "", - $hbase_zookeeper_quorum = "", - $hive_execution_engine = "mr") { + if ("hive-server2" in $roles) { + include hadoop_hive::server2 + + # include hadoop::init_hdfs + # Class['Hadoop::Init_hdfs'] -> Class['Hadoop_hive::Server2'] + # if ("mapred-app" in $roles) { + # Class['Hadoop::Mapred_app'] -> Class['Hadoop_hive::Server2'] + # } + } + } + class client_package { package { "hive": ensure => latest, } + } + + class common_config ($hbase_master = "", + $hbase_zookeeper_quorum = "", + $kerberos_realm = "", + $server2_thrift_port = "10000", + $server2_thrift_http_port = "10001", + $hive_execution_engine = "mr") { + include hadoop_hive::client_package + if ($kerberos_realm) { + require kerberos::client + kerberos::host_keytab { "hive": + spnego => true, + require => Package["hive"], + } + } file { "/etc/hive/conf/hive-site.xml": content => template('hadoop_hive/hive-site.xml'), require => Package["hive"], } } + + class client($hbase_master = "", + $hbase_zookeeper_quorum = "", + $hive_execution_engine = "mr") { + + include hadoop_hive::common_config + } + + class server2 { + include hadoop_hive::common_config + + package { "hive-server2": + ensure => latest, + } + + service { "hive-server2": + ensure => running, + require => Package["hive-server2"], + subscribe => File["/etc/hive/conf/hive-site.xml"], + hasrestart => true, + hasstatus => true, + } + Kerberos::Host_keytab <| title == "hive" |> -> Service["hive-server2"] + } } http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c82c31f/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml b/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml index 2aca237..a973055 100644 --- a/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml +++ b/bigtop-deploy/puppet/modules/hadoop_hive/templates/hive-site.xml @@ -65,4 +65,38 @@ <description>This is the WAR file with the jsp content for Hive Web Interface</description> </property> +<property> + <name>hive.server2.allow.user.substitution</name> + <value>true</value> +</property> + +<property> + <name>hive.server2.enable.doAs</name> + <value>true</value> +</property> + +<property> + <name>hive.server2.thrift.port</name> + <value><%= @server2_thrift_port %></value> +</property> + +<property> + <name>hive.server2.thrift.http.port</name> + <value><%= @server2_thrift_http_port %></value> +</property> + +<% if @kerberos_realm != "" %> +<property> + <name>hive.server2.authentication</name> + <value>KERBEROS</value> +</property> +<property> + <name>hive.server2.authentication.kerberos.principal</name> + <value>hive/_HOST@<%= @kerberos_realm %></value> +</property> +<property> + <name>hive.server2.authentication.kerberos.keytab</name> + <value>/etc/hive.keytab</value> +</property> +<% end %> </configuration>
