Elukey has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/395527 )
Change subject: role::analytics_cluster::hive/oozie: move to profiles
......................................................................
role::analytics_cluster::hive/oozie: move to profiles
Bug: T167790
Change-Id: I21eafba18d6825c1c75f06ee918287249546b7e6
---
D hieradata/common/cdh/hive.yaml
D hieradata/common/cdh/oozie.yaml
D hieradata/common/cdh/oozie/server.yaml
M hieradata/common/profile/hive/client.yaml
A hieradata/common/profile/hive/metastore/database.yaml
A hieradata/common/profile/oozie/client.yaml
M hieradata/role/common/analytics_cluster/coordinator.yaml
R modules/profile/manifests/analytics/database/meta.pp
M modules/profile/manifests/hive/client.pp
R modules/profile/manifests/hive/metastore.pp
R modules/profile/manifests/hive/metastore/database.pp
R modules/profile/manifests/hive/server.pp
M modules/profile/manifests/oozie/client.pp
R modules/profile/manifests/oozie/server.pp
R modules/profile/manifests/oozie/server/database.pp
M modules/role/manifests/analytics_cluster/coordinator.pp
16 files changed, 84 insertions(+), 72 deletions(-)
Approvals:
Elukey: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/common/cdh/hive.yaml b/hieradata/common/cdh/hive.yaml
deleted file mode 100644
index bbff820..0000000
--- a/hieradata/common/cdh/hive.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-#
-# Analytics Cluster Hive Configuration
-#
-cdh::hive::metastore_host: analytics1003.eqiad.wmnet
-
-# The WMF webrequest table uses HCatalog's JSON Serde.
-# Automatically include this in Hive client classpaths.
-cdh::hive::auxpath:
file:///usr/lib/hive-hcatalog/share/hcatalog/hive-hcatalog-core.jar
-
-# Following Cloudera recommandations for ~20 users
-cdh::hive::hive_metastore_opts: '-Xms4g -Xmx10g'
-cdh::hive::hive_server_opts: '-Xms4g -Xmx6g'
\ No newline at end of file
diff --git a/hieradata/common/cdh/oozie.yaml b/hieradata/common/cdh/oozie.yaml
deleted file mode 100644
index 3f1fb38..0000000
--- a/hieradata/common/cdh/oozie.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-#
-# Analytics Cluster Oozie Configuration
-#
-
-cdh::oozie::oozie_host: analytics1003.eqiad.wmnet
\ No newline at end of file
diff --git a/hieradata/common/cdh/oozie/server.yaml
b/hieradata/common/cdh/oozie/server.yaml
deleted file mode 100644
index e43a455..0000000
--- a/hieradata/common/cdh/oozie/server.yaml
+++ /dev/null
@@ -1 +0,0 @@
-cdh::oozie::server::heapsize: 2048
diff --git a/hieradata/common/profile/hive/client.yaml
b/hieradata/common/profile/hive/client.yaml
index b29187f..566e8f3 100644
--- a/hieradata/common/profile/hive/client.yaml
+++ b/hieradata/common/profile/hive/client.yaml
@@ -3,4 +3,9 @@
# check/modify.
profile::hive::client::zookeeper_cluster_name: main-eqiad
profile::hive::client::server_host: analytics1003.eqiad.wmnet
-profile::hive::client::server_port: 10000
\ No newline at end of file
+profile::hive::client::server_port: 10000
+profile::hive::client::metastore_host: 'analytics1003.eqiad.wmnet'
+
+# The WMF webrequest table uses HCatalog's JSON Serde.
+# Automatically include this in Hive client classpaths.
+profile::hive::client::auxpath:
'file:///usr/lib/hive-hcatalog/share/hcatalog/hive-hcatalog-core.jar'
\ No newline at end of file
diff --git a/hieradata/common/profile/hive/metastore/database.yaml
b/hieradata/common/profile/hive/metastore/database.yaml
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/hieradata/common/profile/hive/metastore/database.yaml
diff --git a/hieradata/common/profile/oozie/client.yaml
b/hieradata/common/profile/oozie/client.yaml
new file mode 100644
index 0000000..bbe593b
--- /dev/null
+++ b/hieradata/common/profile/oozie/client.yaml
@@ -0,0 +1 @@
+profile::oozie::client::oozie_host: 'analytics1003.eqiad.wmnet'
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/coordinator.yaml
b/hieradata/role/common/analytics_cluster/coordinator.yaml
index 6a89e1b..582d9cc 100644
--- a/hieradata/role/common/analytics_cluster/coordinator.yaml
+++ b/hieradata/role/common/analytics_cluster/coordinator.yaml
@@ -1,3 +1,13 @@
standard::has_ganglia: false
# Periodical backups of the analytics-meta mysql database.
profile::analytics::database::meta::backup::rsync_dest:
'analytics1002.eqiad.wmnet::backup/mysql/analytics-meta'
+
+profile::hive::server::monitoring_enabled: true
+profile::hive::metastore::monitoring_enabled: true
+profile::analytics::database::meta::monitoring_enabled: true
+profile::oozie::server::monitoring_enabled: true
+
+# Following Cloudera recommandations for ~20 users
+profile::hive::client::hive_metastore_opts: '-Xms4g -Xmx10g'
+profile::hive::client::hive_server_opts: '-Xms4g -Xmx6g'
+profile::oozie::server::jvm_heap_size: 2048
\ No newline at end of file
diff --git a/modules/role/manifests/analytics_cluster/database/meta.pp
b/modules/profile/manifests/analytics/database/meta.pp
similarity index 91%
rename from modules/role/manifests/analytics_cluster/database/meta.pp
rename to modules/profile/manifests/analytics/database/meta.pp
index d7ddbaa..bcda89e 100644
--- a/modules/role/manifests/analytics_cluster/database/meta.pp
+++ b/modules/profile/manifests/analytics/database/meta.pp
@@ -1,12 +1,15 @@
# == Class role::analytics_cluster::database::meta
+#
# Installs a MySQL/MariaDB server for use with Hive and Oozie
# and other Analytics Cluster services.
#
-class role::analytics_cluster::database::meta {
+class profile::analytics::database::meta(
+ $monitoring_enabled =
hiera('profile::analytics::database::meta::monitoring_enabled'),
+) {
# Some CDH database init scripts need Java to run.
require ::profile::java::analytics
- include ::mariadb::packages_wmf
+ class { '::mariadb::packages_wmf': }
$config_template = $::realm ? {
# Production instance has large innodb_buffer_pool_size.
@@ -52,7 +55,7 @@
}
# Include icinga alerts if production realm.
- if $::realm == 'production' {
+ if $monitoring_enabled {
nrpe::monitor_service { 'mysql_analytics-meta':
description => 'analytics-meta MySQL instance',
nrpe_command => '/usr/lib/nagios/plugins/check_procs -c 1:1 -C
mysqld',
diff --git a/modules/profile/manifests/hive/client.pp
b/modules/profile/manifests/hive/client.pp
index c02b370..b8edb60 100644
--- a/modules/profile/manifests/hive/client.pp
+++ b/modules/profile/manifests/hive/client.pp
@@ -7,16 +7,16 @@
$zookeeper_cluster_name =
hiera('profile::hive::client::zookeeper_cluster_name'),
$hiveserver_host = hiera('profile::hive::client::server_host'),
$hiveserver_port = hiera('profile::hive::client::server_port'),
+ $metastore_host =
hiera('profile::hive::client::hive_metastore_host'),
+ $hive_server_opts = hiera('profile::hive::client::hive_server_opts',
undef),
+ $hive_metastore_opts =
hiera('profile::hive::client::hive_metastore_opts', undef),
) {
require ::profile::hadoop::common
# The WMF webrequest table uses HCatalog's JSON Serde.
# Automatically include this in Hive client classpaths.
$hcatalog_jar =
'file:///usr/lib/hive-hcatalog/share/hcatalog/hive-hcatalog-core.jar'
-
$auxpath = $hcatalog_jar
-
-
$zookeeper_hosts =
keys($zookeeper_clusters[$zookeeper_cluster_name]['hosts'])
# You must set at least:
@@ -35,6 +35,9 @@
auxpath => $auxpath,
# default to using Snappy for parquet formatted tables
parquet_compression => 'SNAPPY',
+ hive_server_opts => $hive_server_opts,
+ hive_metastore_opts => $hive_metastore_opts,
+ metastore_host => $metastore_host,
}
# Set up a wrapper script for beeline, the command line
diff --git a/modules/role/manifests/analytics_cluster/hive/metastore.pp
b/modules/profile/manifests/hive/metastore.pp
similarity index 69%
rename from modules/role/manifests/analytics_cluster/hive/metastore.pp
rename to modules/profile/manifests/hive/metastore.pp
index 49cf76c..832094a 100644
--- a/modules/role/manifests/analytics_cluster/hive/metastore.pp
+++ b/modules/profile/manifests/hive/metastore.pp
@@ -1,11 +1,12 @@
-# == Class role::analytics_cluster::hive::metastore
+# == Class profile::hive::metastore
+#
# Sets up Hive Metastore service
#
-# filtertags: labs-project-analytics labs-project-math
-class role::analytics_cluster::hive::metastore {
- system::role { 'analytics_cluster::hive::metastore':
- description => 'hive-metastore service',
- }
+class profile::hive::metastore(
+ $monitoring_enabled =
hiera('profile::hive::metastore::monitoring_enabled', false),
+ $statsd = hiera('statsd'),
+) {
+
require ::profile::hive::client
# Setup hive-metastore
@@ -13,7 +14,7 @@
# Use jmxtrans for sending metrics
class { '::cdh::hive::jmxtrans::metastore':
- statsd => hiera('statsd'),
+ statsd => $statsd,
}
ferm::service{ 'hive_metastore':
@@ -23,7 +24,7 @@
}
# Include icinga alerts if production realm.
- if $::realm == 'production' {
+ if $monitoring_enabled {
nrpe::monitor_service { 'hive-metasore':
description => 'Hive Metastore',
nrpe_command => '/usr/lib/nagios/plugins/check_procs -c 1:1 -C
java -a "org.apache.hadoop.hive.metastore.HiveMetaStore"',
diff --git
a/modules/role/manifests/analytics_cluster/hive/metastore/database.pp
b/modules/profile/manifests/hive/metastore/database.pp
similarity index 82%
rename from modules/role/manifests/analytics_cluster/hive/metastore/database.pp
rename to modules/profile/manifests/hive/metastore/database.pp
index fbf3d14..8e9d3bb 100644
--- a/modules/role/manifests/analytics_cluster/hive/metastore/database.pp
+++ b/modules/profile/manifests/hive/metastore/database.pp
@@ -1,4 +1,5 @@
-# == Class role::analytics_cluster::hive::metastore::database
+# == Class profile::hive::metastore::database
+#
# Includes the role::analytics_cluster::database::meta class
# to install a database for analytics cluster meta data,
# includes the cdh::hive::metastore::mysql
@@ -6,10 +7,10 @@
# and then finally ensures grants and permissions are
# set so that configured hosts can properly connect to this database.
#
-# filtertags: labs-project-analytics labs-project-math
-class role::analytics_cluster::hive::metastore::database {
+class profile::hive::metastore::database {
# Install a database server (MariaDB)
- require ::role::analytics_cluster::database::meta
+ require ::profile::analytics::database::meta
+
# Need to have hive package installed to
# get /usr/lib/hive/bin/schematool.
# require ::profile::hive::client
@@ -22,9 +23,8 @@
# cd /usr/lib/hive/scripts/metastore/upgrade/mysql && sudo mysql
hive_metastore < hive-schema-1.1.0.mysql.sql
# after cdh::hive::metastore::mysql makes puppet fail.
-
class { '::cdh::hive::metastore::mysql':
- require => Class['role::analytics_cluster::database::meta'],
+ require => Class['profile::analytics::database::meta'],
}
# cdh::hive::metastore::mysql only ensures that
diff --git a/modules/role/manifests/analytics_cluster/hive/server.pp
b/modules/profile/manifests/hive/server.pp
similarity index 63%
rename from modules/role/manifests/analytics_cluster/hive/server.pp
rename to modules/profile/manifests/hive/server.pp
index ad6c730..658800c 100644
--- a/modules/role/manifests/analytics_cluster/hive/server.pp
+++ b/modules/profile/manifests/hive/server.pp
@@ -1,19 +1,19 @@
-# == Class role::analytics_cluster::hive::server
-# Sets up Hive Server2
+# == Class profile::hive::server
#
-# filtertags: labs-project-analytics labs-project-math
-class role::analytics_cluster::hive::server {
- system::role { 'analytics_cluster::hive::server':
- description => 'hive-server2 service',
- }
- require ::profile::hive::client
+# Sets up Hive Server2 (no metastore, needs another profile).
+#
+class profile::hive::server(
+ $monitoring_enabled = hiera('profile::hive::server::monitoring_enabled',
false),
+ $statsd = hiera('statsd')
+) {
+ include ::profile::hive::client
# Setup hive-server
class { '::cdh::hive::server': }
# Use jmxtrans for sending metrics
class { '::cdh::hive::jmxtrans::server':
- statsd => hiera('statsd'),
+ statsd => $statsd,
}
ferm::service{ 'hive_server':
@@ -23,7 +23,7 @@
}
# Include icinga alerts if production realm.
- if $::realm == 'production' {
+ if $monitoring_enabled {
nrpe::monitor_service { 'hive-server2':
description => 'Hive Server',
nrpe_command => '/usr/lib/nagios/plugins/check_procs -c 1:1 -C
java -a "org.apache.hive.service.server.HiveServer2"',
diff --git a/modules/profile/manifests/oozie/client.pp
b/modules/profile/manifests/oozie/client.pp
index d3cf6fc..f63c81a 100644
--- a/modules/profile/manifests/oozie/client.pp
+++ b/modules/profile/manifests/oozie/client.pp
@@ -1,6 +1,9 @@
# == Class profile::oozie::client
#
-# filtertags: labs-project-math labs-project-analytics
-class profile::oozie::client {
- class { '::cdh::oozie': }
+class profile::oozie::client(
+ $oozie_host = hiera('profile::oozie::client::oozie_host'),
+) {
+ class { '::cdh::oozie':
+ oozie_host => $oozie_host,
+ }
}
\ No newline at end of file
diff --git a/modules/role/manifests/analytics_cluster/oozie/server.pp
b/modules/profile/manifests/oozie/server.pp
similarity index 83%
rename from modules/role/manifests/analytics_cluster/oozie/server.pp
rename to modules/profile/manifests/oozie/server.pp
index 59d4ecb..ee51854 100644
--- a/modules/role/manifests/analytics_cluster/oozie/server.pp
+++ b/modules/profile/manifests/oozie/server.pp
@@ -1,13 +1,11 @@
# == Class role::analytics_cluster::oozie::server
-# Installs Oozie server
-# Make sure you set hiera variables for cdh::oozie::server appropriately,
-# especially if you are hosting
#
-# filtertags: labs-project-analytics labs-project-math
-class role::analytics_cluster::oozie::server {
- system::role { 'analytics_cluster::oozie::server':
- description => 'Oozie Server',
- }
+# Installs the Oozie server.
+#
+class profile::oozie::server(
+ $monitoring_enabled = hiera('profile::oozie::server::monitoring_enabled',
false),
+ $jvm_heap_size = hiera('profile::oozie::server::jvm_heap_size', 1000),
+) {
require ::profile::oozie::client
# cdh::oozie::server will ensure that its MySQL DB is
@@ -29,6 +27,7 @@
# oozie.service.ProxyUserService.proxyuser.*
# settings look like they are properly configured.
authorization_service_authorization_enabled => false,
+ heapsize => $jvm_heap_size,
}
# Oozie is creating event logs in /var/log/oozie.
@@ -48,7 +47,7 @@
}
# Include icinga alerts if production realm.
- if $::realm == 'production' {
+ if $monitoring_enabled {
nrpe::monitor_service { 'oozie':
description => 'Oozie Server',
nrpe_command => '/usr/lib/nagios/plugins/check_procs -c 1:1 -C
java -a "org.apache.catalina.startup.Bootstrap"',
diff --git a/modules/role/manifests/analytics_cluster/oozie/server/database.pp
b/modules/profile/manifests/oozie/server/database.pp
similarity index 80%
rename from modules/role/manifests/analytics_cluster/oozie/server/database.pp
rename to modules/profile/manifests/oozie/server/database.pp
index 8b5df0c..dd55b7b 100644
--- a/modules/role/manifests/analytics_cluster/oozie/server/database.pp
+++ b/modules/profile/manifests/oozie/server/database.pp
@@ -1,4 +1,5 @@
-# == Class role::analytics_cluster::oozie::server::database
+# == Class profile::oozie::server::database
+#
# Includes the role::analytics_cluster::database::meta class
# to install a database for analytics cluster meta data,
# includes the cdh::oozie::database::mysql
@@ -6,14 +7,13 @@
# and then finally ensures grants and permissions are
# set so that configured hosts can properly connect to this database.
#
-# filtertags: labs-project-analytics labs-project-math
-class role::analytics_cluster::oozie::server::database {
+class profile::oozie::server::database {
# Install a database server (MariaDB)
- require ::role::analytics_cluster::database::meta
+ require ::profile::analytics::database::meta
# Ensure that the oozie db is created.
class { '::cdh::oozie::database::mysql':
- require => Class['role::analytics_cluster::database::meta'],
+ require => Class['profile::analytics::database::meta'],
}
# NOTE: on 2016-02-23, Otto and Joal
diff --git a/modules/role/manifests/analytics_cluster/coordinator.pp
b/modules/role/manifests/analytics_cluster/coordinator.pp
index cfd8a00..e0ff78a 100644
--- a/modules/role/manifests/analytics_cluster/coordinator.pp
+++ b/modules/role/manifests/analytics_cluster/coordinator.pp
@@ -17,21 +17,26 @@
# the Analytics Refinery.
#
class role::analytics_cluster::coordinator {
+
+ system::role { 'analytics_cluster::coordinator':
+ description => 'Analytics Cluster running various Hadoop services
(Hive, Camus, Oozie, ..)'
+ }
+
include ::role::analytics_cluster::client
- include ::role::analytics_cluster::database::meta
+ include ::profile::analytics::database::meta
# Back up analytics-meta MySQL instance
# to analytics1002.
include ::profile::analytics::database::meta::backup
# SQL-like queries to data stored in HDFS
- include ::role::analytics_cluster::hive::metastore
- include ::role::analytics_cluster::hive::server
- include ::role::analytics_cluster::hive::metastore::database
+ include ::profile::hive::metastore
+ include ::profile::hive::server
+ include ::profile::hive::metastore::database
# The Hadoop job scheduler
- include ::role::analytics_cluster::oozie::server
- include ::role::analytics_cluster::oozie::server::database
+ include ::profile::oozie::server
+ include ::profile::oozie::server::database
# Camus crons import data into
# from Kafka into HDFS.
--
To view, visit https://gerrit.wikimedia.org/r/395527
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I21eafba18d6825c1c75f06ee918287249546b7e6
Gerrit-PatchSet: 6
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Elukey <[email protected]>
Gerrit-Reviewer: Elukey <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
