Ottomata has submitted this change and it was merged.
Change subject: Puppetize HA YARN ResourceManager for Hadoop
......................................................................
Puppetize HA YARN ResourceManager for Hadoop
Change-Id: I6c1803c2699dc398d56be779e3d82ad4753926f4
---
M manifests/role/analytics/hadoop.pp
M modules/cdh
2 files changed, 21 insertions(+), 0 deletions(-)
Approvals:
Ottomata: Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/role/analytics/hadoop.pp
b/manifests/role/analytics/hadoop.pp
index bc0c0fb..aca55ac 100644
--- a/manifests/role/analytics/hadoop.pp
+++ b/manifests/role/analytics/hadoop.pp
@@ -13,6 +13,12 @@
# $hadoop_namenodes - Comma separated list of FQDNs that should be NameNodes
# for this cluster. The first entry in the list
# is assumed to be the preferred primary NameNode.
Required.
+# This list will also be used as $resourcemanager_hosts.
+# If hiera('zookeeper_hosts') is set, and this list has
more
+# than one entry, and $journalnode_hosts is also set,
then
+# HA YARN ResourceManager will be configured.
+# TODO: Change the name of this variable to
hadoop_masters
+# When we make this work better with hiera.
#
# $journalnode_hosts - Comma separated list of FQDNs that should be
JournalNodes
# for this cluster. Optional. If not specified, HA
will not be configured.
@@ -61,6 +67,9 @@
# This needs to be set in order to use Impala
$dfs_datanode_hdfs_blocks_metadata_enabled = true
+ # Look up zookeeper_hosts from hiera.
+ $zookeeper_hosts = keys(hiera('zookeeper_hosts', undef))
+
# Configs specific to Production.
if $::realm == 'production' {
# This is the logical name of the Analytics Hadoop cluster.
@@ -70,6 +79,7 @@
'analytics1001.eqiad.wmnet',
'analytics1002.eqiad.wmnet',
]
+ $resourcemanager_hosts = $namenode_hosts
# JournalNodes are colocated on worker DataNodes.
$journalnode_hosts = [
@@ -222,6 +232,7 @@
undef => [$::fqdn],
default => split($::hadoop_namenodes, ','),
}
+ $resourcemanager_hosts = $namenode_hosts
$journalnode_hosts = $::hadoop_journalnodes ? {
undef => undef,
@@ -301,6 +312,8 @@
cluster_name => $cluster_name,
namenode_hosts => $namenode_hosts,
journalnode_hosts => $journalnode_hosts,
+ resourcemanager_hosts => $resourcemanager_hosts,
+ zookeeper_hosts => $zookeeper_hosts,
datanode_mounts => $datanode_mounts,
dfs_name_dir => [$hadoop_name_directory],
dfs_journalnode_edits_dir => $hadoop_journal_directory,
@@ -613,6 +626,14 @@
critical => 'true',
}
}
+
+
+ # If this is a resourcemanager host is set, then go ahead
+ # and include a resourcemanager on all standby nodes as well
+ # as the master node.
+ if $::fqdn in $resourcemanager_hosts {
+ include cdh::hadoop::resourcemanager
+ }
}
diff --git a/modules/cdh b/modules/cdh
index c69732f..3d00a40 160000
--- a/modules/cdh
+++ b/modules/cdh
-Subproject commit c69732fbd3fd561fc910efe79d0168b9366a7d21
+Subproject commit 3d00a4004177e0afd4a123ed365e7a930269a31c
--
To view, visit https://gerrit.wikimedia.org/r/209021
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I6c1803c2699dc398d56be779e3d82ad4753926f4
Gerrit-PatchSet: 5
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
