Elukey has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/370798 )
Change subject: role::analytics_cluster::hadoop::client: move to profiles (p2)
......................................................................
role::analytics_cluster::hadoop::client: move to profiles (p2)
This change will also refactor a lot of analytics roles to profiles
as side effect.
Bug: T167790
Change-Id: Ia2dc439dc885953c78f35425a39bd7b9ed8ef109
---
M hieradata/role/common/analytics_cluster/client.yaml
M hieradata/role/common/analytics_cluster/druid/worker.yaml
D hieradata/role/common/analytics_cluster/druid/zookeeper.yaml
D hieradata/role/common/analytics_cluster/hadoop/client.yaml
M hieradata/role/common/analytics_cluster/hadoop/master.yaml
M hieradata/role/common/analytics_cluster/hadoop/standby.yaml
M hieradata/role/common/analytics_cluster/hadoop/worker.yaml
D hieradata/role/common/analytics_cluster/hive/client.yaml
A hieradata/role/common/analytics_cluster/hive/metastore.yaml
A hieradata/role/common/analytics_cluster/hive/server.yaml
M hieradata/role/common/analytics_cluster/hue.yaml
D hieradata/role/common/analytics_cluster/oozie/client.yaml
D hieradata/role/common/analytics_cluster/refinery.yaml
M manifests/site.pp
R modules/profile/manifests/druid/common.pp
R modules/profile/manifests/hadoop/client.pp
R modules/profile/manifests/hive/client.pp
A modules/profile/manifests/oozie/client.pp
M modules/role/manifests/analytics_cluster/client.pp
D modules/role/manifests/analytics_cluster/druid/hadoop.pp
M modules/role/manifests/analytics_cluster/druid/worker.pp
D modules/role/manifests/analytics_cluster/druid/zookeeper.pp
M modules/role/manifests/analytics_cluster/hadoop/balancer.pp
M modules/role/manifests/analytics_cluster/hadoop/master.pp
M modules/role/manifests/analytics_cluster/hadoop/standby.pp
M modules/role/manifests/analytics_cluster/hadoop/worker.pp
M modules/role/manifests/analytics_cluster/hive/metastore.pp
M modules/role/manifests/analytics_cluster/hive/metastore/database.pp
M modules/role/manifests/analytics_cluster/hive/server.pp
M modules/role/manifests/analytics_cluster/hue.pp
M modules/role/manifests/analytics_cluster/mysql_password.pp
D modules/role/manifests/analytics_cluster/oozie/client.pp
M modules/role/manifests/analytics_cluster/oozie/server.pp
M modules/role/manifests/analytics_cluster/refinery.pp
34 files changed, 130 insertions(+), 117 deletions(-)
Approvals:
Elukey: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/role/common/analytics_cluster/client.yaml
b/hieradata/role/common/analytics_cluster/client.yaml
index 7bc321e..c0d91dc 100644
--- a/hieradata/role/common/analytics_cluster/client.yaml
+++ b/hieradata/role/common/analytics_cluster/client.yaml
@@ -1,5 +1,8 @@
-# FIXME:
-# indirect hiera lookup due to includes in the role:
-# role::analytics_cluster::hadoop::client
-# role::analytics_cluster::hive::client
-hadoop_zookeeper_cluster_name: main-eqiad
\ No newline at end of file
+profile::hadoop::client::zookeeper_cluster_name: main-eqiad
+profile::hadoop::client::resourcemanager_hosts:
+ - analytics1001.eqiad.wmnet
+ - analytics1002.eqiad.wmnet
+
+profile::hive::client::zookeeper_cluster_name: main-eqiad
+profile::hive::client::server_host: analytics1003.eqiad.wmnet
+profile::hive::client::server_port: 10000
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/druid/worker.yaml
b/hieradata/role/common/analytics_cluster/druid/worker.yaml
index 68a1add..9ca785b 100644
--- a/hieradata/role/common/analytics_cluster/druid/worker.yaml
+++ b/hieradata/role/common/analytics_cluster/druid/worker.yaml
@@ -3,7 +3,26 @@
debdeploy::grains:
debdeploy-druid:
value: standard
-# FIXME:
-# indirect hiera lookup due to includes in the role:
-# role::analytics::druid::common
-zookeeper_cluster_name: druid-eqiad
+
+profile::druid::common::zookeeper_cluster_name: druid-eqiad
+
+# Druid nodes get their own Zookeeper cluster to isolate them
+# from the production ones.
+profile::zookeeper::cluster_name: druid-eqiad
+# Don't page if a zookeeper server in this cluster goes down.
+profile::zookeeper::is_critical: false
+# Max number of connections per IP for Zookeeper
+profile::zookeeper::max_client_connections: 1024
+# Default tick_time is 2000ms, this should allow a max
+# of 16 seconds of latency for Zookeeper client sessions.
+# See comments in role::kafka::analytics::broker for more info.
+profile::zookeeper::sync_limit: 8
+# To avoid version conflics with Cloudera zookeeper package, this
+# class manually specifies which debian package version should be installed.
+profile::zookeeper::zookeeper_version: '3.4.5+dfsg-2+deb8u2'
+profile::zookeeper::firewall::srange: '(($DRUID_HOSTS $ANALYTICS_NETWORKS))'
+
+profile::hadoop::client::zookeeper_cluster_name: main-eqiad
+profile::hadoop::client::resourcemanager_hosts:
+ - analytics1001.eqiad.wmnet
+ - analytics1002.eqiad.wmnet
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/druid/zookeeper.yaml
b/hieradata/role/common/analytics_cluster/druid/zookeeper.yaml
deleted file mode 100644
index 1642e41..0000000
--- a/hieradata/role/common/analytics_cluster/druid/zookeeper.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-# Druid nodes get their own Zookeeper cluster to isolate them
-# from the production ones.
-profile::zookeeper::cluster_name: druid-eqiad
-# Don't page if a zookeeper server in this cluster goes down.
-profile::zookeeper::is_critical: false
-# Max number of connections per IP for Zookeeper
-profile::zookeeper::max_client_connections: 1024
-# Default tick_time is 2000ms, this should allow a max
-# of 16 seconds of latency for Zookeeper client sessions.
-# See comments in role::kafka::analytics::broker for more info.
-profile::zookeeper::sync_limit: 8
-# To avoid version conflics with Cloudera zookeeper package, this
-# class manually specifies which debian package version should be installed.
-profile::zookeeper::zookeeper_version: '3.4.5+dfsg-2+deb8u2'
-profile::zookeeper::firewall::srange: '(($DRUID_HOSTS $ANALYTICS_NETWORKS))'
diff --git a/hieradata/role/common/analytics_cluster/hadoop/client.yaml
b/hieradata/role/common/analytics_cluster/hadoop/client.yaml
deleted file mode 100644
index b4d3199..0000000
--- a/hieradata/role/common/analytics_cluster/hadoop/client.yaml
+++ /dev/null
@@ -1 +0,0 @@
-hadoop_zookeeper_cluster_name: main-eqiad
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/hadoop/master.yaml
b/hieradata/role/common/analytics_cluster/hadoop/master.yaml
index 81dde56..60232c9 100644
--- a/hieradata/role/common/analytics_cluster/hadoop/master.yaml
+++ b/hieradata/role/common/analytics_cluster/hadoop/master.yaml
@@ -15,3 +15,8 @@
debdeploy::grains:
debdeploy-hadoop-master:
value: standard
+
+profile::hadoop::client::zookeeper_cluster_name: 'main-eqiad'
+profile::hadoop::client::resourcemanager_hosts:
+ - analytics1001.eqiad.wmnet
+ - analytics1002.eqiad.wmnet
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/hadoop/standby.yaml
b/hieradata/role/common/analytics_cluster/hadoop/standby.yaml
index adf2d84..16a81f6 100644
--- a/hieradata/role/common/analytics_cluster/hadoop/standby.yaml
+++ b/hieradata/role/common/analytics_cluster/hadoop/standby.yaml
@@ -15,3 +15,8 @@
debdeploy::grains:
debdeploy-hadoop-standby:
value: standard
+
+profile::hadoop::client::zookeeper_cluster_name: 'main-eqiad'
+profile::hadoop::client::resourcemanager_hosts:
+ - analytics1001.eqiad.wmnet
+ - analytics1002.eqiad.wmnet
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/hadoop/worker.yaml
b/hieradata/role/common/analytics_cluster/hadoop/worker.yaml
index 0460941..458ab72 100644
--- a/hieradata/role/common/analytics_cluster/hadoop/worker.yaml
+++ b/hieradata/role/common/analytics_cluster/hadoop/worker.yaml
@@ -14,3 +14,12 @@
# NRPE check for them, so the base module's should ignore them.
profile::base::check_disk_options: '-w 6% -c 3% -W 6% -K 3% -l -e -A -i
"/var/lib/hadoop/data"'
profile::base::check_raid_policy: 'WriteBack'
+
+profile::hadoop::client::zookeeper_cluster_name: 'main-eqiad'
+profile::hadoop::client::resourcemanager_hosts:
+ - analytics1001.eqiad.wmnet
+ - analytics1002.eqiad.wmnet
+
+profile::hive::client::zookeeper_cluster_name: main-eqiad
+profile::hive::client::server_host: analytics1003.eqiad.wmnet
+profile::hive::client::server_port: 10000
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/hive/client.yaml
b/hieradata/role/common/analytics_cluster/hive/client.yaml
deleted file mode 100644
index b4d3199..0000000
--- a/hieradata/role/common/analytics_cluster/hive/client.yaml
+++ /dev/null
@@ -1 +0,0 @@
-hadoop_zookeeper_cluster_name: main-eqiad
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/hive/metastore.yaml
b/hieradata/role/common/analytics_cluster/hive/metastore.yaml
new file mode 100644
index 0000000..782fa93
--- /dev/null
+++ b/hieradata/role/common/analytics_cluster/hive/metastore.yaml
@@ -0,0 +1,3 @@
+profile::hive::client::zookeeper_cluster_name: main-eqiad
+profile::hive::client::server_host: analytics1003.eqiad.wmnet
+profile::hive::client::server_port: 10000
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/hive/server.yaml
b/hieradata/role/common/analytics_cluster/hive/server.yaml
new file mode 100644
index 0000000..782fa93
--- /dev/null
+++ b/hieradata/role/common/analytics_cluster/hive/server.yaml
@@ -0,0 +1,3 @@
+profile::hive::client::zookeeper_cluster_name: main-eqiad
+profile::hive::client::server_host: analytics1003.eqiad.wmnet
+profile::hive::client::server_port: 10000
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/hue.yaml
b/hieradata/role/common/analytics_cluster/hue.yaml
index 860725a..3af55f1 100644
--- a/hieradata/role/common/analytics_cluster/hue.yaml
+++ b/hieradata/role/common/analytics_cluster/hue.yaml
@@ -1,8 +1,12 @@
debdeploy::grains:
debdeploy-hue:
value: standard
-# FIXME:
-# indirect hiera lookup due to includes in the role:
-# role::analytics_cluster::hadoop::client
-# role::analytics_cluster::hive::client
-hadoop_zookeeper_cluster_name: main-eqiad
+
+profile::hadoop::client::zookeeper_cluster_name: main-eqiad
+profile::hadoop::client::resourcemanager_hosts:
+ - analytics1001.eqiad.wmnet
+ - analytics1002.eqiad.wmnet
+
+profile::hive::client::zookeeper_cluster_name: main-eqiad
+profile::hive::client::server_host: analytics1003.eqiad.wmnet
+profile::hive::client::server_port: 10000
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/oozie/client.yaml
b/hieradata/role/common/analytics_cluster/oozie/client.yaml
deleted file mode 100644
index e6592c2..0000000
--- a/hieradata/role/common/analytics_cluster/oozie/client.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-# FIXME:
-# indirect hiera lookup due to includes in the role:
-# role::analytics_cluster::hadoop::client
-hadoop_zookeeper_cluster_name: main-eqiad
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/refinery.yaml
b/hieradata/role/common/analytics_cluster/refinery.yaml
deleted file mode 100644
index e6592c2..0000000
--- a/hieradata/role/common/analytics_cluster/refinery.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-# FIXME:
-# indirect hiera lookup due to includes in the role:
-# role::analytics_cluster::hadoop::client
-hadoop_zookeeper_cluster_name: main-eqiad
\ No newline at end of file
diff --git a/manifests/site.pp b/manifests/site.pp
index 0d783cf..64f9d64 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -41,9 +41,7 @@
# - YARN ResourceManager
node 'analytics1001.eqiad.wmnet' {
role(analytics_cluster::hadoop::master,
- analytics_cluster::users,
- # Need druid user and HDFS directories
- analytics_cluster::druid::hadoop)
+ analytics_cluster::users)
include ::standard
include ::base::firewall
@@ -60,9 +58,7 @@
# (MySQL analytics-meta) instance. If you move this,
# make sure /srv/backup/mysql/analytics-meta has
# enough space to store backups.
- analytics_cluster::database::meta::backup_dest,
- # Need druid user and HDFS directories
- analytics_cluster::druid::hadoop)
+ analytics_cluster::database::meta::backup_dest)
include ::standard
include ::base::firewall
@@ -778,9 +774,7 @@
# Analytics Druid servers.
# https://wikitech.wikimedia.org/wiki/Analytics/Data_Lake#Druid
node /^druid100[123].eqiad.wmnet$/ {
- role(analytics_cluster::druid::worker,
- analytics_cluster::hadoop::client,
- analytics_cluster::druid::zookeeper)
+ role(analytics_cluster::druid::worker)
include ::base::firewall
include ::standard
diff --git a/modules/role/manifests/analytics_cluster/druid/common.pp
b/modules/profile/manifests/druid/common.pp
similarity index 72%
rename from modules/role/manifests/analytics_cluster/druid/common.pp
rename to modules/profile/manifests/druid/common.pp
index cc5c83f..3e52095 100644
--- a/modules/role/manifests/analytics_cluster/druid/common.pp
+++ b/modules/profile/manifests/druid/common.pp
@@ -1,4 +1,4 @@
-# == Class role::analytics_cluster::druid::common
+# == Class profile::druid::common
# Installs the druid common package and common configuration settings.
# You will likely not have to include this class directly.
#
@@ -9,13 +9,14 @@
# Druid Zookeeper chroot will be set according to $site in production, or
# $labsproject in labs.
#
-class role::analytics_cluster::druid::common
-{
+class profile::druid::common(
+ $zookeeper_cluster_name =
hiera('profile::druid::common::zookeeper_cluster_name'),
+ $zookeeper_clusters = hiera('zookeeper_clusters'),
+ $druid_properties = hiera('druid::properties'),
+) {
# Need Java before Druid is installed.
require ::profile::java::analytics
- $zookeeper_cluster_name = hiera('zookeeper_cluster_name')
- $zookeeper_clusters = hiera('zookeeper_clusters')
$zookeeper_hosts =
join(keys($zookeeper_clusters[$zookeeper_cluster_name]['hosts']), ',')
$zookeeper_chroot = $::realm ? {
@@ -29,17 +30,13 @@
'druid.zk.service.host' => $zookeeper_hosts,
}
- # Look up druid::properties out of hiera. Since class path
- # lookup does not do hiera hash merging, we do so manually here.
- $hiera_druid_properties = hiera_hash('druid::properties', {})
-
# Druid Common Class
class { '::druid':
# Merge our auto configured zookeeper properties
# with the properties from hiera.
properties => merge(
$zookeeper_properties,
- $hiera_druid_properties
+ $druid_properties
),
}
-}
+}
\ No newline at end of file
diff --git a/modules/role/manifests/analytics_cluster/hadoop/client.pp
b/modules/profile/manifests/hadoop/client.pp
similarity index 89%
rename from modules/role/manifests/analytics_cluster/hadoop/client.pp
rename to modules/profile/manifests/hadoop/client.pp
index 4ade4a0..ce7cfc3 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/client.pp
+++ b/modules/profile/manifests/hadoop/client.pp
@@ -1,8 +1,12 @@
-# == Class role::analytics_cluster::hadoop::client
+# == Class profile::hadoop::client
# Installs Hadoop client pacakges and configuration.
#
# filtertags: labs-project-analytics
-class role::analytics_cluster::hadoop::client {
+class profile::hadoop::client (
+ $zookeeper_clusters = hiera('zookeeper_clusters'),
+ $zookeeper_cluster_name =
hiera('profile::hadoop::client::zookeeper_cluster_name'),
+ $hadoop_resourcemanagers =
hiera('profile::hadoop::client::resourcemanager_hosts'),
+) {
# Include Wikimedia's thirdparty/cloudera apt component
# as an apt source on all Hadoop hosts. This is needed
# to install CDH packages from our apt repo mirror.
@@ -11,7 +15,7 @@
# Force apt-get update to run before we try to install packages.
# CDH Packages are in the thirdparty/cloudera apt component,
# and are made available by profile::cdh::apt.
- Exec['apt-get update'] ->
Class['::role::analytics_cluster::hadoop::client']
+ Exec['apt-get update'] -> Class['::profile::hadoop::client']
# Need Java before Hadoop is installed.
require ::profile::java::analytics
@@ -28,17 +32,12 @@
# zookeeper_clusters
# zookeeper_cluster_name
#
-
- $zookeeper_clusters = hiera('zookeeper_clusters')
- $zookeeper_cluster_name = hiera('hadoop_zookeeper_cluster_name')
- $zookeeper_hosts =
keys($zookeeper_clusters[$zookeeper_cluster_name]['hosts'])
+ $zookeeper_hosts =
keys($zookeeper_clusters[$zookeeper_cluster_name]['hosts'])
class { '::cdh::hadoop':
# Default to using running resourcemanager on the same hosts
# as the namenodes.
- resourcemanager_hosts => hiera(
- 'cdh::hadoop::resourcemanager_hosts',
hiera('cdh::hadoop::namenode_hosts')
- ),
+ resourcemanager_hosts =>
$hadoop_resourcemanagers,
zookeeper_hosts => $zookeeper_hosts,
dfs_name_dir =>
[$hadoop_name_directory],
dfs_journalnode_edits_dir =>
$hadoop_journal_directory,
@@ -84,7 +83,6 @@
}
include ::profile::hadoop::logstash
-
include ::ores::base
diff --git a/modules/role/manifests/analytics_cluster/hive/client.pp
b/modules/profile/manifests/hive/client.pp
similarity index 78%
rename from modules/role/manifests/analytics_cluster/hive/client.pp
rename to modules/profile/manifests/hive/client.pp
index 81c4cb6..769de9d 100644
--- a/modules/role/manifests/analytics_cluster/hive/client.pp
+++ b/modules/profile/manifests/hive/client.pp
@@ -1,9 +1,14 @@
-# == Class role::analytics_cluster::hive::client
+# == Class profile::hive::client
# Installs base configs and packages for hive client nodes.
#
# filtertags: labs-project-analytics labs-project-math
-class role::analytics_cluster::hive::client {
- require ::role::analytics_cluster::hadoop::client
+class profile::hive::client(
+ $zookeeper_clusters = hiera('zookeeper_clusters'),
+ $zookeeper_cluster_name =
hiera('profile::hive::client::zookeeper_cluster_name'),
+ $hiveserver_host = hiera('profile::hive::client::server_host'),
+ $hiveserver_port = hiera('profile::hive::client::server_port'),
+) {
+ require ::profile::hadoop::client
# The WMF webrequest table uses HCatalog's JSON Serde.
# Automatically include this in Hive client classpaths.
@@ -11,8 +16,7 @@
$auxpath = $hcatalog_jar
- $zookeeper_clusters = hiera('zookeeper_clusters')
- $zookeeper_cluster_name = hiera('hadoop_zookeeper_cluster_name')
+
$zookeeper_hosts =
keys($zookeeper_clusters[$zookeeper_cluster_name]['hosts'])
# You must set at least:
@@ -36,9 +40,6 @@
# Set up a wrapper script for beeline, the command line
# interface to HiveServer2 and install it at
# /usr/local/bin/beeline
-
- $hiveserver_host = hiera('hive_server_host', $::cdh::hive::metastore_host)
- $hiveserver_port = hiera('hive_server_port', '10000')
file { '/usr/local/bin/beeline':
content =>
template('role/analytics_cluster/hive/beeline_wrapper.py.erb'),
diff --git a/modules/profile/manifests/oozie/client.pp
b/modules/profile/manifests/oozie/client.pp
new file mode 100644
index 0000000..d3cf6fc
--- /dev/null
+++ b/modules/profile/manifests/oozie/client.pp
@@ -0,0 +1,6 @@
+# == Class profile::oozie::client
+#
+# filtertags: labs-project-math labs-project-analytics
+class profile::oozie::client {
+ class { '::cdh::oozie': }
+}
\ No newline at end of file
diff --git a/modules/role/manifests/analytics_cluster/client.pp
b/modules/role/manifests/analytics_cluster/client.pp
index 37d4eb3..89ccdbd 100644
--- a/modules/role/manifests/analytics_cluster/client.pp
+++ b/modules/role/manifests/analytics_cluster/client.pp
@@ -8,9 +8,10 @@
}
# Include Hadoop ecosystem client classes.
- require ::role::analytics_cluster::hadoop::client
- require ::role::analytics_cluster::hive::client
- require ::role::analytics_cluster::oozie::client
+ require ::profile::hadoop::client
+ require ::profile::hive::client
+ require ::profile::oozie::client
+
# These don't require any extra configuration,
# so no role class is needed.
require ::cdh::pig
diff --git a/modules/role/manifests/analytics_cluster/druid/hadoop.pp
b/modules/role/manifests/analytics_cluster/druid/hadoop.pp
deleted file mode 100644
index 3e7d6cf..0000000
--- a/modules/role/manifests/analytics_cluster/druid/hadoop.pp
+++ /dev/null
@@ -1,9 +0,0 @@
-# == Class role::analytics_cluster::druid::hadoop
-# Ensures that the druid user exists and that
-# druid directories exist in HDFS. This should
-# only be included on Hadoop NameNodes.
-#
-# filtertags: labs-project-analytics
-class role::analytics_cluster::druid::hadoop {
- include ::druid::cdh::hadoop::setup
-}
diff --git a/modules/role/manifests/analytics_cluster/druid/worker.pp
b/modules/role/manifests/analytics_cluster/druid/worker.pp
index 0cfb63e..3a7f047 100644
--- a/modules/role/manifests/analytics_cluster/druid/worker.pp
+++ b/modules/role/manifests/analytics_cluster/druid/worker.pp
@@ -8,8 +8,14 @@
#
# filtertags: labs-project-analytics
class role::analytics_cluster::druid::worker {
+
# Require common druid package and configuration.
- require ::role::analytics_cluster::druid::common
+ require ::profile::druid::common
+ require ::profile::hadoop::client
+
+ # Zookeeper is co-located on the same host
+ include profile::zookeeper::server
+ include profile::zookeeper::firewall
# Auto reload daemons in labs, but not in production.
$should_subscribe = $::realm ? {
diff --git a/modules/role/manifests/analytics_cluster/druid/zookeeper.pp
b/modules/role/manifests/analytics_cluster/druid/zookeeper.pp
deleted file mode 100644
index 581e3fc..0000000
--- a/modules/role/manifests/analytics_cluster/druid/zookeeper.pp
+++ /dev/null
@@ -1,4 +0,0 @@
-class role::analytics_cluster::druid::zookeeper {
- include profile::zookeeper::server
- include profile::zookeeper::firewall
-}
\ No newline at end of file
diff --git a/modules/role/manifests/analytics_cluster/hadoop/balancer.pp
b/modules/role/manifests/analytics_cluster/hadoop/balancer.pp
index b250022..d459474 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/balancer.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/balancer.pp
@@ -1,7 +1,7 @@
# == Class role::analytics_cluster::hadoop::balancer
# Runs hdfs balancer periodically to keep data balanced across all DataNodes
class role::analytics_cluster::hadoop::balancer {
- Class['role::analytics_cluster::hadoop::client'] ->
Class['role::analytics_cluster::hadoop::balancer']
+ require ::profile::hadoop::client
file { '/usr/local/bin/hdfs-balancer':
source =>
'puppet:///modules/role/analytics_cluster/hadoop/hdfs-balancer',
diff --git a/modules/role/manifests/analytics_cluster/hadoop/master.pp
b/modules/role/manifests/analytics_cluster/hadoop/master.pp
index 4e8cce6..6fe82d3 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/master.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/master.pp
@@ -7,9 +7,11 @@
description => 'Hadoop Master (NameNode & ResourceManager)',
}
- require ::role::analytics_cluster::hadoop::client
+ require ::profile::hadoop::client
include ::role::analytics_cluster::monitoring::disks
+ class { '::druid::cdh::hadoop::setup': }
+
class { '::cdh::hadoop::master': }
# Master should run httpfs daemon.
diff --git a/modules/role/manifests/analytics_cluster/hadoop/standby.pp
b/modules/role/manifests/analytics_cluster/hadoop/standby.pp
index 3e03d14..243a7b1 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/standby.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/standby.pp
@@ -6,9 +6,11 @@
system::role { 'analytics_cluster::hadoop::standby':
description => 'Hadoop Standby NameNode',
}
- require ::role::analytics_cluster::hadoop::client
+ require ::profile::hadoop::client
include ::role::analytics_cluster::monitoring::disks
+ class { '::druid::cdh::hadoop::setup': }
+
class { '::cdh::hadoop::namenode::standby': }
# Use jmxtrans for sending metrics
diff --git a/modules/role/manifests/analytics_cluster/hadoop/worker.pp
b/modules/role/manifests/analytics_cluster/hadoop/worker.pp
index 110b503..69c6625 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/worker.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/worker.pp
@@ -7,7 +7,7 @@
description => 'Hadoop Worker (DataNode & NodeManager)',
}
- require ::role::analytics_cluster::hadoop::client
+ require ::profile::hadoop::client
include ::role::analytics_cluster::monitoring::disks
class { '::cdh::hadoop::worker': }
@@ -98,7 +98,7 @@
# This installs hive-hcatalog package on worker nodes to get
# hcatalog jars, including Hive JsonSerde for using
# JSON backed Hive tables.
- include ::role::analytics_cluster::hive::client
+ include ::profile::hive::client
# Spark Python stopped working in Spark 1.5.0 with Oozie,
# for complicated reasons. We need to be able to set
diff --git a/modules/role/manifests/analytics_cluster/hive/metastore.pp
b/modules/role/manifests/analytics_cluster/hive/metastore.pp
index bbccca9..49cf76c 100644
--- a/modules/role/manifests/analytics_cluster/hive/metastore.pp
+++ b/modules/role/manifests/analytics_cluster/hive/metastore.pp
@@ -6,7 +6,7 @@
system::role { 'analytics_cluster::hive::metastore':
description => 'hive-metastore service',
}
- require ::role::analytics_cluster::hive::client
+ require ::profile::hive::client
# Setup hive-metastore
class { '::cdh::hive::metastore': }
diff --git
a/modules/role/manifests/analytics_cluster/hive/metastore/database.pp
b/modules/role/manifests/analytics_cluster/hive/metastore/database.pp
index dc1eb80..fbf3d14 100644
--- a/modules/role/manifests/analytics_cluster/hive/metastore/database.pp
+++ b/modules/role/manifests/analytics_cluster/hive/metastore/database.pp
@@ -12,7 +12,7 @@
require ::role::analytics_cluster::database::meta
# Need to have hive package installed to
# get /usr/lib/hive/bin/schematool.
- # require role::analytics_cluster::hive::client
+ # require ::profile::hive::client
# Ensure that the hive_metastore db is created.
# TODO: In CDH 5.4,
diff --git a/modules/role/manifests/analytics_cluster/hive/server.pp
b/modules/role/manifests/analytics_cluster/hive/server.pp
index 1b7538e..ad6c730 100644
--- a/modules/role/manifests/analytics_cluster/hive/server.pp
+++ b/modules/role/manifests/analytics_cluster/hive/server.pp
@@ -6,7 +6,7 @@
system::role { 'analytics_cluster::hive::server':
description => 'hive-server2 service',
}
- require ::role::analytics_cluster::hive::client
+ require ::profile::hive::client
# Setup hive-server
class { '::cdh::hive::server': }
diff --git a/modules/role/manifests/analytics_cluster/hue.pp
b/modules/role/manifests/analytics_cluster/hue.pp
index 4bfd2eb..7ca174e 100644
--- a/modules/role/manifests/analytics_cluster/hue.pp
+++ b/modules/role/manifests/analytics_cluster/hue.pp
@@ -11,9 +11,10 @@
# have their corresponding clients
# and configs installed.
# Include Hadoop ecosystem client classes.
- require ::role::analytics_cluster::hadoop::client
- require ::role::analytics_cluster::hive::client
- require ::role::analytics_cluster::oozie::client
+ require ::profile::hadoop::client
+ require ::profile::hive::client
+ require ::profile::oozie::client
+
# These don't require any extra configuration,
# so no role class is needed.
require ::cdh::pig
diff --git a/modules/role/manifests/analytics_cluster/mysql_password.pp
b/modules/role/manifests/analytics_cluster/mysql_password.pp
index ffc1ea3..5413015 100644
--- a/modules/role/manifests/analytics_cluster/mysql_password.pp
+++ b/modules/role/manifests/analytics_cluster/mysql_password.pp
@@ -5,7 +5,7 @@
# out of MySQL into Hadoop.
#
class role::analytics_cluster::mysql_password {
- Class['role::analytics_cluster::hadoop::client'] ->
Class['role::analytics_cluster::mysql_password']
+ require ::profile::hadoop::client
include ::passwords::mysql::research
$research_user = $::passwords::mysql::research::user
diff --git a/modules/role/manifests/analytics_cluster/oozie/client.pp
b/modules/role/manifests/analytics_cluster/oozie/client.pp
deleted file mode 100644
index 0b8181b..0000000
--- a/modules/role/manifests/analytics_cluster/oozie/client.pp
+++ /dev/null
@@ -1,8 +0,0 @@
-# == Class role::analytics_cluster::oozie::client
-#
-# filtertags: labs-project-math labs-project-analytics
-class role::analytics_cluster::oozie::client {
- include ::role::analytics_cluster::hadoop::client
-
- class { '::cdh::oozie': }
-}
diff --git a/modules/role/manifests/analytics_cluster/oozie/server.pp
b/modules/role/manifests/analytics_cluster/oozie/server.pp
index 2350d18..59d4ecb 100644
--- a/modules/role/manifests/analytics_cluster/oozie/server.pp
+++ b/modules/role/manifests/analytics_cluster/oozie/server.pp
@@ -8,7 +8,7 @@
system::role { 'analytics_cluster::oozie::server':
description => 'Oozie Server',
}
- require ::role::analytics_cluster::oozie::client
+ require ::profile::oozie::client
# cdh::oozie::server will ensure that its MySQL DB is
# properly initialized. For puppet to do this,
diff --git a/modules/role/manifests/analytics_cluster/refinery.pp
b/modules/role/manifests/analytics_cluster/refinery.pp
index 8c33eb0..21c4001 100644
--- a/modules/role/manifests/analytics_cluster/refinery.pp
+++ b/modules/role/manifests/analytics_cluster/refinery.pp
@@ -7,7 +7,7 @@
# is intended to work with Hadoop, and many of the
# role classes here use the hdfs user, which is created
# by the CDH packages.
- Class['role::analytics_cluster::hadoop::client'] ->
Class['role::analytics_cluster::refinery']
+ require ::profile::hadoop::client
# Clone mediawiki/event-schemas so refinery can use them.
include ::eventschemas
--
To view, visit https://gerrit.wikimedia.org/r/370798
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ia2dc439dc885953c78f35425a39bd7b9ed8ef109
Gerrit-PatchSet: 17
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Elukey <[email protected]>
Gerrit-Reviewer: Elukey <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
