Ottomata has submitted this change and it was merged.
Change subject: Puppetizing Hadoop JournalNode and Standby HA NameNode
......................................................................
Puppetizing Hadoop JournalNode and Standby HA NameNode
Also prepping for Hadoop node repaving
Change-Id: I160b898a547bd575f1126ba93c3b0e509706a65c
---
M manifests/role/analytics.pp
M manifests/role/analytics/hadoop.pp
M manifests/role/analytics/hive.pp
M manifests/role/analytics/oozie.pp
M manifests/site.pp
5 files changed, 94 insertions(+), 181 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
jenkins-bot: Verified
diff --git a/manifests/role/analytics.pp b/manifests/role/analytics.pp
index 778d1ad..644becd 100644
--- a/manifests/role/analytics.pp
+++ b/manifests/role/analytics.pp
@@ -122,112 +122,3 @@
require => Package['libdclass-java'],
}
}
-
-
-
-
-
-### Classes below this line need refactored and/or removed
-### for the repaving of analytics cluster.
-
-
-
-# front end interfaces for Kraken and Hadoop
-class role::analytics::frontend inherits role::analytics {
- # include a mysql database for Sqoop and Oozie
- # with the datadir at /a/mysql
- class { "generic::mysql::server":
- datadir => "/a/mysql",
- version => "5.5",
- }
-}
-
-# Monitoring for kafka brokers.
-class role::analytics::kafka::server inherits role::analytics {
- include misc::analytics::monitoring::kafka::server
-}
-
-
-
-# == role::analytics::udp2log::mobile
-#
-# Geocodes and anonymizes webrequest logs generated on mobile hosts
-# and pipes them into Kafka.
-#
-# NOTE: These instances will be removed once we switch to Kafka 0.8
-# and start producing logs from varnish frontend hosts.
-#
-class role::analytics::udp2log::mobile {
- include misc::udp2log,
- misc::udp2log::utilities,
- role::cache::configuration,
- passwords::analytics
-
- misc::udp2log::instance { 'analytics-mobile':
- multicast => true,
- monitor_log_age => false,
- logrotate_template => 'udp2log/logrotate_udp2log_analytics.erb',
- }
-
- misc::analytics::monitoring::kafka::producer { 'webrequest-mobile':
- jmx_port => 9951,
- ganglia => '239.192.1.32:8649',
- produce_events_warning => 2000000,
- produce_events_critical => 1000000,
- }
-}
-
-# == role::analytics::udp2log::wikipedia_mobile
-#
-# Pipes webrequest logs generated on mobile hosts into Kafka.
-# This stream is not geocoded or anonymized.
-#
-# NOTE: These instances will be removed once we switch to Kafka 0.8
-# and start producing logs from varnish frontend hosts.
-#
-class role::analytics::udp2log::wikipedia_mobile {
- include misc::udp2log,
- misc::udp2log::utilities,
- role::cache::configuration
-
- misc::udp2log::instance { 'analytics-wikipedia-mobile':
- multicast => true,
- monitor_log_age => false,
- logrotate_template => 'udp2log/logrotate_udp2log_analytics.erb',
- }
-
- misc::analytics::monitoring::kafka::producer {
'webrequest-wikipedia-mobile':
- jmx_port => 9951,
- ganglia => '239.192.1.32:8649',
- produce_events_warning => 2000000,
- produce_events_critical => 1000000,
- }
-}
-
-
-# == role::analytics::udp2log::sampled
-#
-# Geocodes and anonymizses a full sampled 1000 stream
-# and pipes it into Kafka.
-#
-# NOTE: These instances will be removed once we switch to Kafka 0.8
-# and start producing logs from varnish frontend hosts.
-#
-class role::analytics::udp2log::sampled {
- include misc::udp2log,
- misc::udp2log::utilities,
- passwords::analytics
-
- misc::udp2log::instance { 'analytics-sampled':
- multicast => true,
- monitor_log_age => false,
- logrotate_template => 'udp2log/logrotate_udp2log_analytics.erb',
- }
-
- misc::analytics::monitoring::kafka::producer {
'webrequest-all-sampled-1000':
- jmx_port => 9954,
- ganglia => '239.192.1.32:8649',
- produce_events_warning => 80,
- produce_events_critical => 40,
- }
-}
diff --git a/manifests/role/analytics/hadoop.pp
b/manifests/role/analytics/hadoop.pp
index 817c8ec..2b73846 100644
--- a/manifests/role/analytics/hadoop.pp
+++ b/manifests/role/analytics/hadoop.pp
@@ -50,6 +50,12 @@
include cdh4::hadoop::worker
}
+# == Class role::analytics::hadoop::standby inherits
role::analytics::hadoop::client
+#
+class role::analytics::hadoop::standby inherits
role::analytics::hadoop::client {
+ system_role { 'role::analytics::hadoop::standby': description => 'Hadoop
Standby NameNode' }
+ include cdh4::hadoop::namenode::standby
+}
### The following classes should not be included directly.
@@ -62,10 +68,26 @@
# Common hadoop configs for the production Kraken cluster
#
class role::analytics::hadoop::production {
- $namenode_hostname = 'analytics1010.eqiad.wmnet'
- $hadoop_name_directory = '/var/lib/hadoop/name'
+ # This is the logical name of the Analytics Hadoop cluster.
+ $nameservice_id = 'kraken'
+ $namenode_hosts = [
+ 'analytics1010.eqiad.wmnet',
+ 'analytics1009.eqiad.wmnet',
+ ]
+ # I'm not sure if running JournalNodes
+ # on DataNode hosts is a good or bad idea.
+ # Doing this here now for lack of a better place.
+ $journalnode_hosts = [
+ 'analytics1012.eqiad.wmnet',
+ 'analytics1014.eqiad.wmnet',
+ 'analytics1016.eqiad.wmnet',
+ ]
+
+ $hadoop_name_directory = '/var/lib/hadoop/name'
$hadoop_data_directory = '/var/lib/hadoop/data'
+ $hadoop_journal_directory = '/var/lib/hadoop/journal'
+
$datanode_mounts = [
"$hadoop_data_directory/c",
"$hadoop_data_directory/d",
@@ -80,7 +102,9 @@
]
class { 'cdh4::hadoop':
- namenode_hostname => $namenode_hostname,
+ namenode_hosts => $namenode_hosts,
+ nameservice_id => $nameservice_id,
+ journalnode_hosts => $journalnode_hosts,
datanode_mounts => $datanode_mounts,
dfs_name_dir => [$hadoop_name_directory],
dfs_block_size => 268435456, # 256 MB
@@ -122,13 +146,24 @@
# this nodes $::fqdn.
#
class role::analytics::hadoop::labs {
- # if the global variable $::hadoop_namenode is set,
- # use it as the namenode_hostname. This allows
+ # if the global variable $::hadoop_namenodes is set,
+ # use it as the namenode_hostnames. This allows
# configuration via the Labs Instance configuration page.
- $namenode_hostname = $::hadoop_namenode ? {
- undef => $::fqdn,
- default => "${::hadoop_namenode}.${domain}",
+ $namenode_hosts = $::hadoop_namenodes ? {
+ undef => [$::fqdn],
+ default => split($::hadoop_namenodes, ','),
}
+
+ $journalnode_hosts = $::hadoop_journalnodes ? {
+ undef => undef,
+ default => split($::hadoop_journalnodes, ','),
+ }
+
+ $nameservice_id = $::hadoop_nameservice_id ? {
+ undef => undef,
+ default => $::hadoop_nameservice_id,
+ }
+
$hadoop_name_directory = '/var/lib/hadoop/name'
@@ -138,6 +173,8 @@
"$hadoop_data_directory/b",
]
+ $hadoop_journal_directory = '/var/lib/hadoop/journal'
+
# We don't have to create any partions in labs, so it
# is unlikely that /var/lib/hadoop and $hadoop_data_directory
# will be created manually. Ensure it and datanode_mounts exist.
@@ -146,9 +183,12 @@
}
class { 'cdh4::hadoop':
- namenode_hostname => $namenode_hostname,
+ namenode_hosts => $namenode_hosts,
+ nameservice_id => $nameservice_id,
+ journalnode_hosts => $journalnode_hosts,
datanode_mounts => $datanode_mounts,
dfs_name_dir => [$hadoop_name_directory],
+ dfs_journalnode_edits_dir => $hadoop_journal_directory,
dfs_block_size => 268435456, # 256 MB
io_file_buffer_size => 131072,
mapreduce_map_tasks_maximum => 2,
diff --git a/manifests/role/analytics/hive.pp b/manifests/role/analytics/hive.pp
index bfdd70d..6155f01 100644
--- a/manifests/role/analytics/hive.pp
+++ b/manifests/role/analytics/hive.pp
@@ -72,7 +72,7 @@
#
class role::analytics::hive::labs {
class { '::cdh4::hive':
- metastore_host => $role::analytics::hadoop::labs::namenode_hostname,
+ metastore_host => $role::analytics::hadoop::labs::namenode_hosts[0],
zookeeper_hosts => $role::analytics::zookeeper::hosts_array,
}
}
\ No newline at end of file
diff --git a/manifests/role/analytics/oozie.pp
b/manifests/role/analytics/oozie.pp
index dae2561..7cb680e 100644
--- a/manifests/role/analytics/oozie.pp
+++ b/manifests/role/analytics/oozie.pp
@@ -6,7 +6,7 @@
$oozie_host = $::realm ? {
'production' => 'analytics1027.eqiad.wmnet',
- 'labs' => $role::analytics::hadoop::labs::namenode_hostname,
+ 'labs' => $role::analytics::hadoop::labs::namenode_hosts[0],
}
class { 'cdh4::oozie':
diff --git a/manifests/site.pp b/manifests/site.pp
index 3d8f5c2..ab9287d 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -123,15 +123,45 @@
-# analytics1001.wikimedia.org is the analytics cluster master.
+# analytics1001.wikimedia.org has a public IP
node "analytics1001.wikimedia.org" {
include role::analytics
}
-# analytics1011-analytics1020 are Hadoop Worker nodes
-node /analytics10(1[7-9]|20).eqiad.wmnet/ {
- include role::analytics::common
- include role::analytics::hadoop::worker
+# Unassigned Cisco nodes.
+# NOTE: analytics1002-analytics1004 still need reinstalled as of 2013-08-26
+node /analytics100[2-8]\.eqiad\.wmnet/ {
+ include standard
+}
+
+# analytics1009 is the Hadoop standby NameNode
+node "analytics1009.eqiad.wmnet" {
+ include standard
+ # include role::analytics::common
+ # include role::analytics::hadoop::standby
+}
+
+# analytics1010 is the Hadoop master node
+# (primary NameNode, ResourceManager, etc.)
+node "analytics1010.eqiad.wmnet" {
+ include standard
+ # include role::analytics::common
+ # include role::analytics::hadoop::master
+}
+
+
+# analytics1011-analytics1020 are Hadoop worker nodes
+node /analytics10(1[1-9]|20).eqiad.wmnet/ {
+ include standard
+ # include role::analytics::common
+ # include role::analytics::hadoop::worker
+}
+
+# analytics1021 and analytics1022 are Kafka Brokers.
+# NOTE: These still need reinstalled as of 2013-08-26
+node /analytics102[12]\.eqiad\.wmnet/ {
+ include standard
+ # include role::analytics::kafka::server
}
# analytics1023-1025 are zookeeper server nodes
@@ -140,69 +170,21 @@
include role::analytics::zookeeper::server
}
-
-
-
-
-### Analytics nodes below this line need to be reinstalled.
-
-node "analytics1002.eqiad.wmnet" {
- include role::analytics
+# NOTE: This still needs reinstalled as of 2013-08-26
+node "analytics1026.eqiad.wmnet" {
+ include standard
}
-
-#### analytics udp2log kafka producer instances:
-# NOTE: These instances will be removed once we switch to Kafka 0.8
-# and start producing logs from varnish frontend hosts.
-
-node "analytics1006.eqiad.wmnet" {
- include role::analytics
- include role::analytics::udp2log::wikipedia_mobile
-}
-node "analytics1008.eqiad.wmnet" {
- include role::analytics
- include role::analytics::udp2log::sampled
-}
-node "analytics1009.eqiad.wmnet" {
- include role::analytics
- include role::analytics::udp2log::mobile
-}
-
-# analytics1021 and analytics1022 are Kafka Brokers.
-node /analytics102[12]\.eqiad\.wmnet/ {
- include role::analytics::kafka::server
-}
-
-# misc unpuppetized analytics hosts
-node /analytics10(0[3457]|1[0-7]|2[267])\.eqiad\.wmnet/ {
- # ganglia aggregator for the Analytics cluster.
- if ($hostname == 'analytics1003' or $hostname == 'analytics1011') {
- $ganglia_aggregator = true
- }
-
- include role::analytics
-}
-
-
# analytics1027 hosts the frontend
# interfaces to Kraken and Hadoop.
+# NOTE: This still needs reinstalled as of 2013-08-26
node "analytics1027.eqiad.wmnet" {
- include role::analytics::frontend
-
- # Sync files from HDFS /wmf/public to stats.wikimedia.org hourly.
- # This allows Limn and other tools to graph analyized datasets generated
by Kraken.
- # Note: This is a temporary solution until analytics and kraken
- # have been reviewed an puppetized in production operations/puppet.
- misc::analytics::hdfs::sync { "kraken-public":
- hdfs_source => "/wmf/public",
- rsync_destination =>
"stat1001.wikimedia.org::a/srv/stats.wikimedia.org/htdocs/kraken-public",
- }
-
- # check webrequest loss in Kraken HDFS.
- # (This can run on any analytics node.)
- include misc::monitoring::kraken::loss
+ include standard
}
+
+
+
# git.wikimedia.org
node "antimony.wikimedia.org" {
install_certificate{ "git.wikimedia.org": ca => "RapidSSL_CA.pem" }
--
To view, visit https://gerrit.wikimedia.org/r/76722
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I160b898a547bd575f1126ba93c3b0e509706a65c
Gerrit-PatchSet: 8
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
