Elukey has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/370798 )

Change subject: role::analytics_cluster::hadoop::client: move to profiles (p2)
......................................................................

role::analytics_cluster::hadoop::client: move to profiles (p2)

Bug: T167790
Change-Id: Ia2dc439dc885953c78f35425a39bd7b9ed8ef109
---
M hieradata/role/common/analytics_cluster/client.yaml
M hieradata/role/common/analytics_cluster/hue.yaml
M hieradata/role/common/analytics_cluster/oozie/client.yaml
M hieradata/role/common/analytics_cluster/refinery.yaml
M manifests/site.pp
R modules/profile/manifests/hadoop/client.pp
M modules/role/manifests/analytics_cluster/client.pp
M modules/role/manifests/analytics_cluster/druid/worker.pp
M modules/role/manifests/analytics_cluster/hadoop/balancer.pp
M modules/role/manifests/analytics_cluster/hadoop/master.pp
M modules/role/manifests/analytics_cluster/hadoop/standby.pp
M modules/role/manifests/analytics_cluster/hadoop/worker.pp
M modules/role/manifests/analytics_cluster/hive/client.pp
M modules/role/manifests/analytics_cluster/hue.pp
M modules/role/manifests/analytics_cluster/mysql_password.pp
M modules/role/manifests/analytics_cluster/oozie/client.pp
M modules/role/manifests/analytics_cluster/refinery.pp
17 files changed, 27 insertions(+), 42 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/98/370798/1

diff --git a/hieradata/role/common/analytics_cluster/client.yaml 
b/hieradata/role/common/analytics_cluster/client.yaml
index 7bc321e..1936c1d 100644
--- a/hieradata/role/common/analytics_cluster/client.yaml
+++ b/hieradata/role/common/analytics_cluster/client.yaml
@@ -1,5 +1 @@
-# FIXME:
-# indirect hiera lookup due to includes in the role:
-# role::analytics_cluster::hadoop::client
-# role::analytics_cluster::hive::client
-hadoop_zookeeper_cluster_name: main-eqiad
\ No newline at end of file
+profile::hadoop::client::zookeeper_cluster_name: main-eqiad
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/hue.yaml 
b/hieradata/role/common/analytics_cluster/hue.yaml
index 860725a..f723d80 100644
--- a/hieradata/role/common/analytics_cluster/hue.yaml
+++ b/hieradata/role/common/analytics_cluster/hue.yaml
@@ -1,8 +1,4 @@
 debdeploy::grains:
   debdeploy-hue:
     value: standard
-# FIXME:
-# indirect hiera lookup due to includes in the role:
-# role::analytics_cluster::hadoop::client
-# role::analytics_cluster::hive::client
-hadoop_zookeeper_cluster_name: main-eqiad
+profile::hadoop::client::zookeeper_cluster_name: main-eqiad
diff --git a/hieradata/role/common/analytics_cluster/oozie/client.yaml 
b/hieradata/role/common/analytics_cluster/oozie/client.yaml
index e6592c2..1936c1d 100644
--- a/hieradata/role/common/analytics_cluster/oozie/client.yaml
+++ b/hieradata/role/common/analytics_cluster/oozie/client.yaml
@@ -1,4 +1 @@
-# FIXME:
-# indirect hiera lookup due to includes in the role:
-# role::analytics_cluster::hadoop::client
-hadoop_zookeeper_cluster_name: main-eqiad
\ No newline at end of file
+profile::hadoop::client::zookeeper_cluster_name: main-eqiad
\ No newline at end of file
diff --git a/hieradata/role/common/analytics_cluster/refinery.yaml 
b/hieradata/role/common/analytics_cluster/refinery.yaml
index e6592c2..1936c1d 100644
--- a/hieradata/role/common/analytics_cluster/refinery.yaml
+++ b/hieradata/role/common/analytics_cluster/refinery.yaml
@@ -1,4 +1 @@
-# FIXME:
-# indirect hiera lookup due to includes in the role:
-# role::analytics_cluster::hadoop::client
-hadoop_zookeeper_cluster_name: main-eqiad
\ No newline at end of file
+profile::hadoop::client::zookeeper_cluster_name: main-eqiad
\ No newline at end of file
diff --git a/manifests/site.pp b/manifests/site.pp
index 1ddb1c0..b365bb6 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -778,9 +778,7 @@
 # Analytics Druid servers.
 # https://wikitech.wikimedia.org/wiki/Analytics/Data_Lake#Druid
 node /^druid100[123].eqiad.wmnet$/ {
-    role(analytics_cluster::druid::worker,
-        analytics_cluster::hadoop::client,
-        analytics_cluster::druid::zookeeper)
+    role(analytics_cluster::druid::worker)
 
     include ::base::firewall
     include ::standard
diff --git a/modules/role/manifests/analytics_cluster/hadoop/client.pp 
b/modules/profile/manifests/hadoop/client.pp
similarity index 89%
rename from modules/role/manifests/analytics_cluster/hadoop/client.pp
rename to modules/profile/manifests/hadoop/client.pp
index 54a0e42..de42182 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/client.pp
+++ b/modules/profile/manifests/hadoop/client.pp
@@ -1,8 +1,12 @@
-# == Class role::analytics_cluster::hadoop::client
+# == Class profile::hadoop::client
 # Installs Hadoop client pacakges and configuration.
 #
 # filtertags: labs-project-analytics
-class role::analytics_cluster::hadoop::client {
+class profile::hadoop::client (
+    $zookeeper_clusters      = hiera('zookeeper_clusters')
+    $zookeeper_cluster_name  = 
hiera('profile::hadoop::client::zookeeper_cluster_name')
+    $hadoop_resourcemanagers = 
hiera('profile::hadoop::client::resourcemanager_hosts')
+) {
     # Include Wikimedia's thirdparty/cloudera apt component
     # as an apt source on all Hadoop hosts.  This is needed
     # to install CDH packages from our apt repo mirror.
@@ -11,7 +15,7 @@
     # Force apt-get update to run before we try to install packages.
     # CDH Packages are in the thirdparty/cloudera apt component,
     # and are made available by profile::cdh::apt.
-    Exec['apt-get update'] -> 
Class['::role::analytics_cluster::hadoop::client']
+    Exec['apt-get update'] -> Class['::profile::hadoop::client']
 
     # Need Java before Hadoop is installed.
     require ::profile::java::analytics
@@ -28,17 +32,12 @@
     #  zookeeper_clusters
     #  zookeeper_cluster_name
     #
-
-    $zookeeper_clusters     = hiera('zookeeper_clusters')
-    $zookeeper_cluster_name = hiera('hadoop_zookeeper_cluster_name')
-    $zookeeper_hosts        = 
keys($zookeeper_clusters[$zookeeper_cluster_name]['hosts'])
+    $zookeeper_hosts = 
keys($zookeeper_clusters[$zookeeper_cluster_name]['hosts'])
 
     class { '::cdh::hadoop':
         # Default to using running resourcemanager on the same hosts
         # as the namenodes.
-        resourcemanager_hosts                       => hiera(
-            'cdh::hadoop::resourcemanager_hosts', 
hiera('cdh::hadoop::namenode_hosts')
-        ),
+        resourcemanager_hosts                       => 
$hadoop_resourcemanagers,
         zookeeper_hosts                             => $zookeeper_hosts,
         dfs_name_dir                                => 
[$hadoop_name_directory],
         dfs_journalnode_edits_dir                   => 
$hadoop_journal_directory,
@@ -89,7 +88,7 @@
     # NOTE: This conditional is temporary, and will be removed once stat1002
     # is replaced with a Jessie box as part of T159838.
     if os_version('debian >= jessie') {
-        include ::ores::base
+        class { '::ores::base': }
     }
 
     # If in production AND the current node is a journalnode, then
diff --git a/modules/role/manifests/analytics_cluster/client.pp 
b/modules/role/manifests/analytics_cluster/client.pp
index 37d4eb3..c2375dd 100644
--- a/modules/role/manifests/analytics_cluster/client.pp
+++ b/modules/role/manifests/analytics_cluster/client.pp
@@ -8,7 +8,7 @@
     }
 
     # Include Hadoop ecosystem client classes.
-    require ::role::analytics_cluster::hadoop::client
+    require ::profile::hadoop::client
     require ::role::analytics_cluster::hive::client
     require ::role::analytics_cluster::oozie::client
     # These don't require any extra configuration,
diff --git a/modules/role/manifests/analytics_cluster/druid/worker.pp 
b/modules/role/manifests/analytics_cluster/druid/worker.pp
index 0cfb63e..5900bfe 100644
--- a/modules/role/manifests/analytics_cluster/druid/worker.pp
+++ b/modules/role/manifests/analytics_cluster/druid/worker.pp
@@ -10,6 +10,8 @@
 class role::analytics_cluster::druid::worker {
     # Require common druid package and configuration.
     require ::role::analytics_cluster::druid::common
+    require ::profile::hadoop::client
+    require ::role::analytics_cluster::zookeeper
 
     # Auto reload daemons in labs, but not in production.
     $should_subscribe = $::realm ? {
diff --git a/modules/role/manifests/analytics_cluster/hadoop/balancer.pp 
b/modules/role/manifests/analytics_cluster/hadoop/balancer.pp
index b250022..d459474 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/balancer.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/balancer.pp
@@ -1,7 +1,7 @@
 # == Class role::analytics_cluster::hadoop::balancer
 # Runs hdfs balancer periodically to keep data balanced across all DataNodes
 class role::analytics_cluster::hadoop::balancer {
-    Class['role::analytics_cluster::hadoop::client'] -> 
Class['role::analytics_cluster::hadoop::balancer']
+    require ::profile::hadoop::client
 
     file { '/usr/local/bin/hdfs-balancer':
         source => 
'puppet:///modules/role/analytics_cluster/hadoop/hdfs-balancer',
diff --git a/modules/role/manifests/analytics_cluster/hadoop/master.pp 
b/modules/role/manifests/analytics_cluster/hadoop/master.pp
index 4e8cce6..711128d 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/master.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/master.pp
@@ -7,7 +7,7 @@
         description => 'Hadoop Master (NameNode & ResourceManager)',
     }
 
-    require ::role::analytics_cluster::hadoop::client
+    require ::profile::hadoop::client
     include ::role::analytics_cluster::monitoring::disks
 
     class { '::cdh::hadoop::master': }
diff --git a/modules/role/manifests/analytics_cluster/hadoop/standby.pp 
b/modules/role/manifests/analytics_cluster/hadoop/standby.pp
index 3e03d14..38bd2cc 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/standby.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/standby.pp
@@ -6,7 +6,7 @@
     system::role { 'analytics_cluster::hadoop::standby':
         description => 'Hadoop Standby NameNode',
     }
-    require ::role::analytics_cluster::hadoop::client
+    require ::profile::hadoop::client
     include ::role::analytics_cluster::monitoring::disks
 
     class { '::cdh::hadoop::namenode::standby': }
diff --git a/modules/role/manifests/analytics_cluster/hadoop/worker.pp 
b/modules/role/manifests/analytics_cluster/hadoop/worker.pp
index 110b503..3d7c87b 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/worker.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/worker.pp
@@ -7,7 +7,7 @@
         description => 'Hadoop Worker (DataNode & NodeManager)',
     }
 
-    require ::role::analytics_cluster::hadoop::client
+    require ::profile::hadoop::client
     include ::role::analytics_cluster::monitoring::disks
 
     class { '::cdh::hadoop::worker': }
diff --git a/modules/role/manifests/analytics_cluster/hive/client.pp 
b/modules/role/manifests/analytics_cluster/hive/client.pp
index 81c4cb6..efc64eb 100644
--- a/modules/role/manifests/analytics_cluster/hive/client.pp
+++ b/modules/role/manifests/analytics_cluster/hive/client.pp
@@ -3,7 +3,7 @@
 #
 # filtertags: labs-project-analytics labs-project-math
 class role::analytics_cluster::hive::client {
-    require ::role::analytics_cluster::hadoop::client
+    require ::profile::hadoop::client
 
     # The WMF webrequest table uses HCatalog's JSON Serde.
     # Automatically include this in Hive client classpaths.
diff --git a/modules/role/manifests/analytics_cluster/hue.pp 
b/modules/role/manifests/analytics_cluster/hue.pp
index 4bfd2eb..cf3bb7ea 100644
--- a/modules/role/manifests/analytics_cluster/hue.pp
+++ b/modules/role/manifests/analytics_cluster/hue.pp
@@ -11,7 +11,7 @@
     # have their corresponding clients
     # and configs installed.
     # Include Hadoop ecosystem client classes.
-    require ::role::analytics_cluster::hadoop::client
+    require ::profile::hadoop::client
     require ::role::analytics_cluster::hive::client
     require ::role::analytics_cluster::oozie::client
     # These don't require any extra configuration,
diff --git a/modules/role/manifests/analytics_cluster/mysql_password.pp 
b/modules/role/manifests/analytics_cluster/mysql_password.pp
index ffc1ea3..8a2c29c 100644
--- a/modules/role/manifests/analytics_cluster/mysql_password.pp
+++ b/modules/role/manifests/analytics_cluster/mysql_password.pp
@@ -5,7 +5,7 @@
 # out of MySQL into Hadoop.
 #
 class role::analytics_cluster::mysql_password {
-    Class['role::analytics_cluster::hadoop::client'] -> 
Class['role::analytics_cluster::mysql_password']
+    Class['profile::hadoop::client'] -> 
Class['role::analytics_cluster::mysql_password']
 
     include ::passwords::mysql::research
     $research_user = $::passwords::mysql::research::user
diff --git a/modules/role/manifests/analytics_cluster/oozie/client.pp 
b/modules/role/manifests/analytics_cluster/oozie/client.pp
index 0b8181b..3383423 100644
--- a/modules/role/manifests/analytics_cluster/oozie/client.pp
+++ b/modules/role/manifests/analytics_cluster/oozie/client.pp
@@ -2,7 +2,7 @@
 #
 # filtertags: labs-project-math labs-project-analytics
 class role::analytics_cluster::oozie::client {
-    include ::role::analytics_cluster::hadoop::client
+    include ::profile::hadoop::client
 
     class { '::cdh::oozie': }
 }
diff --git a/modules/role/manifests/analytics_cluster/refinery.pp 
b/modules/role/manifests/analytics_cluster/refinery.pp
index 8c33eb0..507686a 100644
--- a/modules/role/manifests/analytics_cluster/refinery.pp
+++ b/modules/role/manifests/analytics_cluster/refinery.pp
@@ -7,7 +7,7 @@
     # is intended to work with Hadoop, and many of the
     # role classes here use the hdfs user, which is created
     # by the CDH packages.
-    Class['role::analytics_cluster::hadoop::client'] -> 
Class['role::analytics_cluster::refinery']
+    require profile::hadoop::client
 
     # Clone mediawiki/event-schemas so refinery can use them.
     include ::eventschemas

-- 
To view, visit https://gerrit.wikimedia.org/r/370798
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia2dc439dc885953c78f35425a39bd7b9ed8ef109
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Elukey <ltosc...@wikimedia.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to