Elukey has uploaded a new change for review.
https://gerrit.wikimedia.org/r/277984
Change subject: Add automatic failover to Hadoop Namenodes.
......................................................................
Add automatic failover to Hadoop Namenodes.
Hadoop namenodes can be configured for automatic failover if HA is enabled.
This requires Zookeeper and a new daemon (hadoop-hdfs-zkfc) that periodically
executes healtch checks and decides what Namenode is the active master.
Bug: T129838
Change-Id: I0a5c34c130909d3f4a058800b09970de65c29596
---
M manifests/hadoop/namenode.pp
M templates/hadoop/core-site.xml.erb
M templates/hadoop/hdfs-site.xml.erb
3 files changed, 42 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet/cdh
refs/changes/84/277984/1
diff --git a/manifests/hadoop/namenode.pp b/manifests/hadoop/namenode.pp
index fbc150a..71d5151 100644
--- a/manifests/hadoop/namenode.pp
+++ b/manifests/hadoop/namenode.pp
@@ -16,6 +16,12 @@
ensure => 'installed',
}
+ if ($::cdh::hadoop::ha_enabled and $::cdh::hadoop::zookeeper_hosts) {
+ package { 'hadoop-hdfs-zkfc':
+ ensure => 'installed',
+ }
+ }
+
# NameNodes expect that the hosts.exclude file exists.
# I don't want to manage this as a puppet file resource,
# as users of this class might want to manage it themselves.
@@ -53,4 +59,26 @@
alias => 'namenode',
require => Exec['hadoop-namenode-format'],
}
+
+
+ if ($::cdh::hadoop::ha_enabled and $::cdh::hadoop::zookeeper_hosts) {
+ # Create a znode in ZooKeeper inside of which the automatic failover
+ # system stores its data.
+ exec { 'hadoop-hdfs-zkfc-init'
+ command => '/usr/bin/hdfs zkfc -formatZK',
+ user => 'hdfs',
+ require => Service['hadoop-hdfs-namenode'],
+ }
+
+ # Supporting daemon to enable automatic-failover via health-check.
+ # Stores its state in zookeper.
+ service { 'hadoop-hdfs-zkfc':
+ ensure => 'running',
+ enable => true,
+ hasstatus => true,
+ hasrestart => true,
+ require => [Exec['hadoop-hdfs-zkfc-init'],
+ Service['hadoop-hdfs-namenode']],
+ }
+ }
}
diff --git a/templates/hadoop/core-site.xml.erb
b/templates/hadoop/core-site.xml.erb
index 1e8af34..a8df776 100644
--- a/templates/hadoop/core-site.xml.erb
+++ b/templates/hadoop/core-site.xml.erb
@@ -12,6 +12,13 @@
<value>hdfs://<%= @ha_enabled ? @nameservice_id : @primary_namenode_host
%>/</value>
</property>
+<% if @ha_enabled and @zookeeper_hosts %>
+ <property>
+ <name>ha.zookeeper.quorum</name>
+ <value><%= Array(@zookeeper_hosts).sort.join(',') %></value>
+ </property>
+<% end -%>
+
<% if @io_file_buffer_size -%>
<property>
<name>io.file.buffer.size</name>
diff --git a/templates/hadoop/hdfs-site.xml.erb
b/templates/hadoop/hdfs-site.xml.erb
index 3ccba3e..d27b76b 100644
--- a/templates/hadoop/hdfs-site.xml.erb
+++ b/templates/hadoop/hdfs-site.xml.erb
@@ -21,6 +21,13 @@
<value>hadoop</value>
</property>
+<% if @ha_enabled and @zookeeper_hosts %>
+ <property>
+ <name>dfs.ha.automatic-failover.enabled</name>
+ <value>true</value>
+ </property>
+<% end -%>
+
<% if @ha_enabled -%>
<property>
<name>dfs.nameservices</name>
--
To view, visit https://gerrit.wikimedia.org/r/277984
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I0a5c34c130909d3f4a058800b09970de65c29596
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet/cdh
Gerrit-Branch: master
Gerrit-Owner: Elukey <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
