Elukey has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/401927 )
Change subject: Refactor thorium's roles in one
......................................................................
Refactor thorium's roles in one
Bug: T167790
Change-Id: I0decd3c62aab104c76915eacad1ab3339107a7f3
---
D hieradata/common/cdh/hue.yaml
R hieradata/role/common/analytics_cluster/webserver.yaml
D hieradata/role/common/druid/analytics/pivot.yaml
M manifests/site.pp
A modules/profile/manifests/druid/pivot.pp
A modules/profile/manifests/hue.pp
M modules/profile/manifests/statistics/web.pp
D modules/role/manifests/analytics_cluster/hue.pp
A modules/role/manifests/analytics_cluster/webserver.pp
D modules/role/manifests/druid/analytics/pivot.pp
D modules/role/manifests/statistics/web.pp
11 files changed, 113 insertions(+), 107 deletions(-)
Approvals:
Ottomata: Looks good to me, but someone else must approve
Elukey: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/common/cdh/hue.yaml b/hieradata/common/cdh/hue.yaml
deleted file mode 100644
index f6b8a7b..0000000
--- a/hieradata/common/cdh/hue.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-#
-# Analytics Cluster Hue Configuration
-#
-
-# Don't auto create Hue users from LDAP in production.
-cdh::hue::ldap_create_users_on_login: false
-
-# External Database configuration
-cdh::hue::database_host: 'analytics1003.eqiad.wmnet'
-cdh::hue::database_engine: 'mysql'
-cdh::hue::database_user: 'hue'
-cdh::hue::database_port: '3306'
-cdh::hue::database_name: 'hue'
\ No newline at end of file
diff --git a/hieradata/role/common/statistics/web.yaml
b/hieradata/role/common/analytics_cluster/webserver.yaml
similarity index 66%
rename from hieradata/role/common/statistics/web.yaml
rename to hieradata/role/common/analytics_cluster/webserver.yaml
index e786679..64c5135 100644
--- a/hieradata/role/common/statistics/web.yaml
+++ b/hieradata/role/common/analytics_cluster/webserver.yaml
@@ -9,3 +9,9 @@
profile::superset::database_uri:
mysql://superset@analytics1003.eqiad.wmnet/superset
profile::superset::workers: 8
profile::superset::ldap_proxy_enabled: true
+
+profile::hue::database_host: 'analytics1003.eqiad.wmnet'
+profile::hue::hive_server_host: 'analytics1003.eqiad.wmnet'
+profile::hue::monitoring_enabled: true
+
+profile::druid::pivot::druid_broker: 'druid1001.eqiad.wmnet:8082'
\ No newline at end of file
diff --git a/hieradata/role/common/druid/analytics/pivot.yaml
b/hieradata/role/common/druid/analytics/pivot.yaml
deleted file mode 100644
index c8049db..0000000
--- a/hieradata/role/common/druid/analytics/pivot.yaml
+++ /dev/null
@@ -1 +0,0 @@
-pivot::druid_broker: 'druid1001.eqiad.wmnet:8082'
\ No newline at end of file
diff --git a/manifests/site.pp b/manifests/site.pp
index bf2a65a..acfad3b 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -2047,13 +2047,7 @@
# related role/module.
#
# This node is not intended for data processing.
- role(statistics::web,
- druid::analytics::pivot,
- analytics_cluster::hue)
-
-
- include ::standard
- include ::base::firewall
+ role(analytics_cluster::webserver)
}
# Failoid service (Ganeti VM)
diff --git a/modules/profile/manifests/druid/pivot.pp
b/modules/profile/manifests/druid/pivot.pp
new file mode 100644
index 0000000..85997b9
--- /dev/null
+++ b/modules/profile/manifests/druid/pivot.pp
@@ -0,0 +1,11 @@
+# Class: profile::druid::pivot
+#
+# Install and configure the Druid's Pivot nodejs UI
+#
+class profile::druid::pivot(
+ $druid_broker = hiera('profile::druid::pivot::druid_broker'),
+) {
+ class { 'pivot':
+ druid_broker => $druid_broker,
+ }
+}
\ No newline at end of file
diff --git a/modules/profile/manifests/hue.pp b/modules/profile/manifests/hue.pp
new file mode 100644
index 0000000..6a12058
--- /dev/null
+++ b/modules/profile/manifests/hue.pp
@@ -0,0 +1,78 @@
+# == Class profile::hue
+#
+# Installs Hue server.
+#
+class profile::hue (
+ $hive_server_host = hiera('profile::hue::hive_server_host'),
+ $database_host = hiera('profile::hue::database_host'),
+ $database_engine = hiera('profile::hue::database_engine',
'mysql'),
+ $database_user = hiera('profile::hue::database_user', 'hue'),
+ $database_port = hiera('profile::hue::database_port', 3306),
+ $database_name = hiera('profile::hue::database_name', 'hue'),
+ $ldap_create_users_on_login =
hiera('profile::hue::ldap_create_users_on_login', false),
+ $monitoring_enabled = hiera('profile::hue::monitoring_enabled',
false),
+){
+
+ # Require that all Hue applications
+ # have their corresponding clients
+ # and configs installed.
+ # Include Hadoop ecosystem client classes.
+ require ::profile::hadoop::common
+ require ::profile::hive::client
+ require ::profile::oozie::client
+
+ # These don't require any extra configuration,
+ # so no role class is needed.
+ class { '::cdh::pig': }
+ class { '::cdh::sqoop': }
+ class { '::cdh::mahout': }
+ class { '::cdh::spark': }
+
+ # LDAP Labs config is the same as LDAP in production.
+ include ::ldap::role::config::labs
+
+ class { '::cdh::hue':
+ # We always host hive-server on the same node as hive-metastore.
+ hive_server_host => $hive_server_host,
+ smtp_host => $::mail_smarthost[0],
+ database_host => $database_host,
+ database_user => $database_user,
+ database_engine => $database_engine,
+ database_name => $database_name,
+ database_port => $database_port,
+ smtp_from_email => "hue@${::fqdn}",
+ ldap_url => inline_template('<%=
scope.lookupvar("ldap::role::config::labs::servernames").collect { |host|
"ldaps://#{host}" }.join(" ") %>'),
+ ldap_bind_dn =>
$ldap::role::config::labs::ldapconfig['proxyagent'],
+ ldap_bind_password =>
$ldap::role::config::labs::ldapconfig['proxypass'],
+ ldap_base_dn => $ldap::role::config::labs::basedn,
+ ldap_username_pattern =>
'uid=<username>,ou=people,dc=wikimedia,dc=org',
+ ldap_user_filter => 'objectclass=person',
+ ldap_user_name_attr => 'uid',
+ ldap_group_filter => 'objectclass=posixgroup',
+ ldap_group_member_attr => 'member',
+ ldap_create_users_on_login => $ldap_create_users_on_login,
+ # Disable hue's SSL. SSL terminiation is handled by an upstream proxy.
+ ssl_private_key => false,
+ ssl_certificate => false,
+ secure_proxy_ssl_header => true,
+ }
+
+ ferm::service{ 'hue_server':
+ proto => 'tcp',
+ port => '8888',
+ srange => '$PRODUCTION_NETWORKS',
+ }
+
+ # Include icinga alerts if production realm.
+ if $monitoring_enabled {
+ nrpe::monitor_service { 'hue':
+ description => 'Hue Server',
+ nrpe_command => '/usr/lib/nagios/plugins/check_procs -c 1:1 -C
python2.7 -a "/usr/lib/hue/build/env/bin/hue"',
+ contact_group => 'admins,analytics',
+ require => Class['cdh::hue'],
+ }
+ }
+}
+
+# TODO: Hue database backup.
+# TODO: Make Hue use MySQL database. Maybe?
diff --git a/modules/profile/manifests/statistics/web.pp
b/modules/profile/manifests/statistics/web.pp
index 888ad08..252d9ef 100644
--- a/modules/profile/manifests/statistics/web.pp
+++ b/modules/profile/manifests/statistics/web.pp
@@ -4,8 +4,6 @@
$statistics_servers = hiera('statistics_servers'),
$geowiki_host = hiera('profile::statistics::web::geowiki_host'),
) {
- include ::standard
- include ::base::firewall
include ::deployment::umask_wikidev
diff --git a/modules/role/manifests/analytics_cluster/hue.pp
b/modules/role/manifests/analytics_cluster/hue.pp
deleted file mode 100644
index 83a972e..0000000
--- a/modules/role/manifests/analytics_cluster/hue.pp
+++ /dev/null
@@ -1,67 +0,0 @@
-# == Class role::analytics_cluster::hue
-# Installs Hue server.
-#
-# filtertags: labs-project-analytics
-class role::analytics_cluster::hue {
- system::role { 'analytics_cluster::hue':
- description => 'Hue (Hadoop User Experience) WebGUI',
- }
-
- # Require that all Hue applications
- # have their corresponding clients
- # and configs installed.
- # Include Hadoop ecosystem client classes.
- require ::profile::hadoop::common
- require ::profile::hive::client
- require ::profile::oozie::client
-
- # These don't require any extra configuration,
- # so no role class is needed.
- require ::cdh::pig
- require ::cdh::sqoop
- require ::cdh::mahout
- require ::cdh::spark
-
- # LDAP Labs config is the same as LDAP in production.
- include ::ldap::role::config::labs
-
- class { '::cdh::hue':
- # We always host hive-server on the same node as hive-metastore.
- hive_server_host => hiera('cdh::hive::metastore_host'),
- smtp_host => $::mail_smarthost[0],
- smtp_from_email => "hue@${::fqdn}",
- ldap_url => inline_template('<%=
scope.lookupvar("ldap::role::config::labs::servernames").collect { |host|
"ldaps://#{host}" }.join(" ") %>'),
- ldap_bind_dn =>
$ldap::role::config::labs::ldapconfig['proxyagent'],
- ldap_bind_password =>
$ldap::role::config::labs::ldapconfig['proxypass'],
- ldap_base_dn => $ldap::role::config::labs::basedn,
- ldap_username_pattern =>
'uid=<username>,ou=people,dc=wikimedia,dc=org',
- ldap_user_filter => 'objectclass=person',
- ldap_user_name_attr => 'uid',
- ldap_group_filter => 'objectclass=posixgroup',
- ldap_group_member_attr => 'member',
- # ldap_create_users_on_login => $ldap_create_users_on_login,
- # Disable hue's SSL. SSL terminiation is handled by an upstream proxy.
- ssl_private_key => false,
- ssl_certificate => false,
- secure_proxy_ssl_header => true,
- }
-
- ferm::service{ 'hue_server':
- proto => 'tcp',
- port => '8888',
- srange => '$PRODUCTION_NETWORKS',
- }
-
- # Include icinga alerts if production realm.
- if $::realm == 'production' {
- nrpe::monitor_service { 'hue':
- description => 'Hue Server',
- nrpe_command => '/usr/lib/nagios/plugins/check_procs -c 1:1 -C
python2.7 -a "/usr/lib/hue/build/env/bin/hue"',
- contact_group => 'admins,analytics',
- require => Class['cdh::hue'],
- }
- }
-}
-
-# TODO: Hue database backup.
-# TODO: Make Hue use MySQL database. Maybe?
diff --git a/modules/role/manifests/analytics_cluster/webserver.pp
b/modules/role/manifests/analytics_cluster/webserver.pp
new file mode 100644
index 0000000..b4d9ffc
--- /dev/null
+++ b/modules/role/manifests/analytics_cluster/webserver.pp
@@ -0,0 +1,17 @@
+class role::analytics_cluster::webserver {
+
+ system::role { 'analytics_cluster::webserver':
+ description => 'Webserver hosting the main Analytics websites'
+ }
+
+ include ::profile::statistics::web
+
+ # Superset. T166689
+ include ::profile::superset
+
+ include ::profile::hue
+ include ::profile::druid::pivot
+
+ include ::profile::base::firewall
+ class { 'standard': }
+}
\ No newline at end of file
diff --git a/modules/role/manifests/druid/analytics/pivot.pp
b/modules/role/manifests/druid/analytics/pivot.pp
deleted file mode 100644
index acebd0d..0000000
--- a/modules/role/manifests/druid/analytics/pivot.pp
+++ /dev/null
@@ -1,6 +0,0 @@
-# == Class role::druid::analytics::pivot
-# Imply's Pivot nodejs UI to explore Druid data
-#
-class role::druid::analytics::pivot {
- include ::pivot
-}
diff --git a/modules/role/manifests/statistics/web.pp
b/modules/role/manifests/statistics/web.pp
deleted file mode 100644
index dda3959..0000000
--- a/modules/role/manifests/statistics/web.pp
+++ /dev/null
@@ -1,11 +0,0 @@
-# (thorium)
-class role::statistics::web {
- system::role { 'statistics::web':
- description => 'Statistics private data host and general compute node'
- }
-
- include ::profile::statistics::web
-
- # Superset. T166689
- include ::profile::superset
-}
--
To view, visit https://gerrit.wikimedia.org/r/401927
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I0decd3c62aab104c76915eacad1ab3339107a7f3
Gerrit-PatchSet: 5
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Elukey <ltosc...@wikimedia.org>
Gerrit-Reviewer: Elukey <ltosc...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: Ottomata <ao...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
