Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/73208
Change subject: Some slight restructuring of analytics role classes to adapt
for hive server role.
......................................................................
Some slight restructuring of analytics role classes to
adapt for hive server role.
Change-Id: I128080e9643ad3c04796c42f4a72a84508ba861b
---
M manifests/role/analytics/hadoop.pp
M manifests/role/analytics/hive.pp
M manifests/role/analytics/zookeeper.pp
3 files changed, 116 insertions(+), 41 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/08/73208/1
diff --git a/manifests/role/analytics/hadoop.pp
b/manifests/role/analytics/hadoop.pp
index 8bb7433..817c8ec 100644
--- a/manifests/role/analytics/hadoop.pp
+++ b/manifests/role/analytics/hadoop.pp
@@ -21,7 +21,7 @@
# == Class role::analytics::hadoop
# Installs base configs for Hadoop nodes
#
-class role::analytics::hadoop {
+class role::analytics::hadoop::client {
# need java before hadoop is installed
require role::analytics::java
@@ -38,17 +38,24 @@
# == Class role::analytics::hadoop::master
# Includes cdh4::hadoop::master classes
#
-class role::analytics::hadoop::master inherits role::analytics::hadoop {
+class role::analytics::hadoop::master inherits role::analytics::hadoop::client
{
system_role { 'role::analytics::hadoop::master': description => 'Hadoop
Master (NameNode & ResourceManager)' }
include cdh4::hadoop::master
}
# == Class role::analytics::hadoop::worker
# Includes cdh4::hadoop::worker classes
-class role::analytics::hadoop::worker inherits role::analytics::hadoop {
+class role::analytics::hadoop::worker inherits role::analytics::hadoop::client
{
system_role { 'role::analytics::hadoop::worker': description => 'Hadoop
Worker (DataNode & NodeManager)' }
include cdh4::hadoop::worker
}
+
+
+
+### The following classes should not be included directly.
+### You should either include role::analytics::hadoop::client,
+### or role::analytics::hadoop::worker or
+### role::analytics::hadoop::master.
# == Class role::analytics::hadoop::production
@@ -115,7 +122,9 @@
# this nodes $::fqdn.
#
class role::analytics::hadoop::labs {
- # if the globa
+ # if the global variable $::hadoop_namenode is set,
+ # use it as the namenode_hostname. This allows
+ # configuration via the Labs Instance configuration page.
$namenode_hostname = $::hadoop_namenode ? {
undef => $::fqdn,
default => "${::hadoop_namenode}.${domain}",
diff --git a/manifests/role/analytics/hive.pp b/manifests/role/analytics/hive.pp
index f96ed1a..46a723a 100644
--- a/manifests/role/analytics/hive.pp
+++ b/manifests/role/analytics/hive.pp
@@ -1,5 +1,74 @@
-# == Class role::analytics::hive
+# role/analytics/hive.pp
#
-class role::analytics::hive {
- include cdh4::hive
+# Role classes for Analytics Hive client and server nodes.
+# These role classes will configure Hive properly in either
+# the Analytics labs or Analytics production environments.
+#
+# These classes require role::analytics::hadoop::client
+# has already been applied. They infer some of their
+# configurations from that role.
+#
+
+
+# == Class role::analytics::hive
+# Installs base configs for hive client nodes
+#
+class role::analytics::hive::client {
+ # require zookeeper config to get zookeeper hosts array.
+ require role::analytics::zookeeper::config
+ Class['role::analytics::hadoop::client'] ->
Class['role::analytics::hive::client']
+
+ # include common labs or production hadoop configs
+ # based on $::realm
+ if ($::realm == 'labs') {
+ include role::analytics::hive::labs
+ }
+ else {
+ include role::analytics::hive::production
+ }
+}
+
+
+
+# == Class role::analytics::hive::server
+# Sets up hive server and hive metastore.
+#
+class role::analytics::hive::server inherits role::analytics::hive::client {
+ if (!defined(Package['mysql-server'])) {
+ package { 'mysql-server':
+ ensure => 'installed',
+ }
+ }
+ class { '::cdh4::hive::master':
+ require => Package['mysql-server'],
+ }
+}
+
+
+
+### The following classes should not be included directly.
+### You should either include role::analytics::hive::client
+### or role::analytics::hive::server.
+
+# == Class role::analytics::hive::production
+# Installs and configures hive for WMF production environment.
+#
+class role::analytics::hive::production {
+ include passwords::analytics
+
+ class { '::cdh4::hive':
+ metastore_host => 'analytics1027.eqiad.wmnet',
+ jdbc_password => $passwords::analytics::hive::jdbc_password,
+ zookeeper_hosts => $role::analytics::zookeeper::hosts_array,
+ }
+}
+
+# == Class role::analytics::hive::labs
+# Installs and configures hive for WMF Labs environment.
+#
+class role::analytics::hive::labs {
+ class { '::cdh4::hive':
+ metastore_host => $role::analytics::hadoop::labs::namenode_hostname,
+ zookeeper_hosts => $role::analytics::zookeeper::hosts_array,
+ }
}
\ No newline at end of file
diff --git a/manifests/role/analytics/zookeeper.pp
b/manifests/role/analytics/zookeeper.pp
index 97d56ba..e839ca0 100644
--- a/manifests/role/analytics/zookeeper.pp
+++ b/manifests/role/analytics/zookeeper.pp
@@ -13,51 +13,48 @@
# include role::analytics::zookeeper::server
#
-
-# == Class role::analytics::zookeeper::client
+# == Class role::analytics::zookeeper::config
+# Bare config role class for client and server classes.
+# You may include this class manually if you need to know use the
+# $role::analytics::zookeeper::hosts or
+# $role::analytics::zookeeper::hosts_array variables.
#
-class role::analytics::zookeeper::client {
- # include common labs or production zookeeper configs
- # based on $::realm
- if ($::realm == 'labs') {
- include role::analytics::zookeeper::labs
+class role::analytics::zookeeper::config {
+ #
+ $labs_hosts = {
+ 'kraken-puppet.pmtpa.wmflabs' => 1,
}
- else {
- include role::analytics::zookeeper::production
- }
-}
-class role::analytics::zookeeper::server inherits
role::analytics::zookeeper::client {
- class { '::zookeeper::server': }
-}
-
-
-# == Class role::analytics::zookeeper::production
-#
-class role::analytics::zookeeper::production {
- $zookeeper_hosts = {
+ $production_hosts = {
'analytics1023.eqiad.wmnet' => 23,
'analytics1024.eqiad.wmnet' => 24,
'analytics1025.eqiad.wmnet' => 25,
}
+ $hosts = $::realm ? {
+ 'labs' => $labs_hosts,
+ 'production' => $production_hosts,
+ }
+
+ # maintain a $hosts_array variable here for
+ # cases where you need a list of zookeeper hosts,
+ # rather than a hash with ZK IDs. (This is used
+ # in role/analytics/hive.pp, for example.)
+ $hosts_array = keys($hosts)
+}
+
+
+# == Class role::analytics::zookeeper::client
+#
+class role::analytics::zookeeper::client {
+ require role::analytics::zookeeper::config
+
class { '::zookeeper':
- hosts => $zookeeper_hosts,
+ hosts => $role::analytics::zookeeper::config::hosts,
version => '3.3.5+dfsg1-1ubuntu1',
}
}
-# == Class role::analytics::zookeeper::labs
-#
-class role::analytics::zookeeper::labs {
- # TODO: make this configurable via a global
- # variable in labs.
- $zookeeper_hosts = {
- 'kraken-zookeeper.pmtpa.wmflabs' => 1,
- }
-
- class { '::zookeeper':
- hosts => $zookeeper_hosts,
- version => '3.3.5+dfsg1-1ubuntu1',
- }
+class role::analytics::zookeeper::server inherits
role::analytics::zookeeper::client {
+ class { '::zookeeper::server': }
}
--
To view, visit https://gerrit.wikimedia.org/r/73208
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I128080e9643ad3c04796c42f4a72a84508ba861b
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
