Repository: bigtop Updated Branches: refs/heads/master 3e17db895 -> e2974452f
BIGTOP-1662: puppet: Fix configuration file incompleteness The changes of BIGTOP-1634 omit kerberos_realm and hadoop_security_authentication variables from some classes so that they're not correctly entered in the generated configuration files. Also, a hasty "fix" to hadoop-env.sh causes all variables to remain unset always. Signed-off-by: Evans Ye <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/e2974452 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/e2974452 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/e2974452 Branch: refs/heads/master Commit: e2974452fb4bf42763c07059becc3daf7542fb99 Parents: 3e17db8 Author: Michael Weiser <[email protected]> Authored: Wed Feb 11 17:11:15 2015 +0100 Committer: Evans Ye <[email protected]> Committed: Thu Feb 12 16:51:43 2015 +0000 ---------------------------------------------------------------------- .../puppet/hieradata/bigtop/cluster.yaml | 4 +++- .../puppet/modules/hadoop/manifests/init.pp | 21 ++++++++++++++------ .../modules/hadoop/templates/hadoop-env.sh | 8 ++++---- 3 files changed, 22 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/e2974452/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml index 41c8e31..3a73a97 100644 --- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml +++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml @@ -49,6 +49,9 @@ #kerberos::site::admin_port: "749" #kerberos::site::keytab_export_dir: "/var/lib/bigtop_keytabs" +# applies to hdfs, yarn, mapred and httpfs +hadoop::kerberos_realm: "%{hiera('kerberos::site::realm')}" + hadoop::common_hdfs::hadoop_namenode_host: "%{hiera('bigtop::hadoop_head_node')}" # actually default but needed for hadoop_namenode_uri here hadoop::common_hdfs::hadoop_namenode_port: "8020" @@ -57,7 +60,6 @@ hadoop::common_yarn::hadoop_ps_host: "%{hiera('bigtop::hadoop_head_node')}" hadoop::common_yarn::hadoop_rm_host: "%{hiera('bigtop::hadoop_head_node')}" # actually default but needed for hue::server::rm_port here hadoop::common_yarn::hadoop_rm_port: "8032" -hadoop::common_yarn::kerberos_realm: "%{hiera('kerberos::site::realm')}" hadoop::common_mapred_app::hadoop_hs_host: "%{hiera('bigtop::hadoop_head_node')}" hadoop::common_mapred_app::hadoop_jobtracker_host: "%{hiera('bigtop::hadoop_head_node')}" http://git-wip-us.apache.org/repos/asf/bigtop/blob/e2974452/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp index 8ee5386..3c4ddd2 100644 --- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp +++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp @@ -14,6 +14,7 @@ # limitations under the License. class hadoop ($hadoop_security_authentication = "simple", + $kerberos_realm = undef, $zk = "", # Set from facter if available $hadoop_storage_dirs = split($::hadoop_storage_dirs, ";"), @@ -89,7 +90,7 @@ class hadoop ($hadoop_security_authentication = "simple", class common_yarn ( $yarn_data_dirs = suffix($hadoop::hadoop_storage_dirs, "/yarn"), - $kerberos_realm = undef, + $kerberos_realm = $hadoop::kerberos_realm, $hadoop_ps_host, $hadoop_ps_port = "20888", $hadoop_rm_host, @@ -156,7 +157,9 @@ class hadoop ($hadoop_security_authentication = "simple", $hadoop_security_group_mapping = undef, $hadoop_core_proxyusers = $hadoop::proxyusers, $hadoop_snappy_codec = undef, - $hadoop_security_authentication = $hadoop::hadoop_security_authentication ) inherits hadoop { + $hadoop_security_authentication = $hadoop::hadoop_security_authentication, + $kerberos_realm = $hadoop::kerberos_realm, + ) inherits hadoop { $sshfence_keydir = "$hadoop_ha_sshfence_user_home/.ssh" $sshfence_keypath = "$sshfence_keydir/id_sshfence" @@ -244,8 +247,10 @@ class hadoop ($hadoop_security_authentication = "simple", $hadoop_mapred_jobtracker_plugins = "", $hadoop_mapred_tasktracker_plugins = "", $mapred_acls_enabled = undef, - $mapred_data_dirs = suffix($hadoop::hadoop_storage_dirs, "/mapred")) { - + $mapred_data_dirs = suffix($hadoop::hadoop_storage_dirs, "/mapred"), + $hadoop_security_authentication = $hadoop::hadoop_security_authentication, + $kerberos_realm = $hadoop::kerberos_realm, + ) inherits hadoop { include common_hdfs package { "hadoop-mapreduce": @@ -265,7 +270,9 @@ class hadoop ($hadoop_security_authentication = "simple", } } - class datanode { + class datanode ( + $hadoop_security_authentication = $hadoop::hadoop_security_authentication, + ) inherits hadoop { include common_hdfs package { "hadoop-hdfs-datanode": @@ -299,7 +306,9 @@ class hadoop ($hadoop_security_authentication = "simple", class httpfs ($hadoop_httpfs_port = "14000", $secret = "hadoop httpfs secret", $hadoop_core_proxyusers = $hadoop::proxyusers, - $hadoop_security_authentcation = $hadoop::hadoop_security_authentication ) inherits hadoop { + $hadoop_security_authentcation = $hadoop::hadoop_security_authentication, + $kerberos_realm = $hadoop::kerberos_realm, + ) inherits hadoop { if ($hadoop_security_authentication == "kerberos") { kerberos::host_keytab { "httpfs": http://git-wip-us.apache.org/repos/asf/bigtop/blob/e2974452/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh index f2e355b..d460c32 100644 --- a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh +++ b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh @@ -15,11 +15,11 @@ <% def shell_config(shell_var, *puppet_var) puppet_var = puppet_var[0] || shell_var.downcase - if @puppet_var - return "export #{shell_var}=#{scope.lookupvar(puppet_var)}" - else - return "#export #{shell_var}=" + puppet_vars = scope.to_hash + if puppet_vars[puppet_var] + return "export #{shell_var}=#{puppet_vars[puppet_var]}" end + return "#export #{shell_var}=" end %> # WARNING: Heavy puppet machinery is involved managing this file, # your edits stand no chance
