Elukey has submitted this change and it was merged. ( 
https://gerrit.wikimedia.org/r/370187 )

Change subject: role::an_cluster::hadoop::client: moving to profiles (first 
part)
......................................................................


role::an_cluster::hadoop::client: moving to profiles (first part)

The idea of this commit is to slowly start moving to profiles
and remove the indirect hiera lookups in our code (all tagged
with FIXME).

This change also introduces new namespaces under profiles,
namely cdh and hadoop, that in my opinion respect a bit more
our dependencies. It also adds a specific profile for all
the Java Analytics depencencies, included in various clusters
(Hadoop, Druid, etc..).

Bug: T167790
Change-Id: Ic3982c73b80693a81036057cbc510e13a2984c5f
---
A hieradata/eqiad/profile/hadoop/logstash.yaml
R modules/profile/manifests/cdh/apt.pp
R modules/profile/manifests/hadoop/logstash.pp
R modules/profile/manifests/java/analytics.pp
R modules/profile/templates/hadoop/hadoop-yarn-logging-helper.erb
M modules/role/manifests/analytics_cluster/database/meta.pp
M modules/role/manifests/analytics_cluster/druid/common.pp
M modules/role/manifests/analytics_cluster/hadoop/client.pp
8 files changed, 27 insertions(+), 20 deletions(-)

Approvals:
  Elukey: Looks good to me, approved
  jenkins-bot: Verified



diff --git a/hieradata/eqiad/profile/hadoop/logstash.yaml 
b/hieradata/eqiad/profile/hadoop/logstash.yaml
new file mode 100644
index 0000000..962cd8a
--- /dev/null
+++ b/hieradata/eqiad/profile/hadoop/logstash.yaml
@@ -0,0 +1 @@
+profile::hadoop::logstash::enabled: false
\ No newline at end of file
diff --git a/modules/role/manifests/analytics_cluster/apt.pp 
b/modules/profile/manifests/cdh/apt.pp
similarity index 67%
rename from modules/role/manifests/analytics_cluster/apt.pp
rename to modules/profile/manifests/cdh/apt.pp
index 514189f..9ba9396 100644
--- a/modules/role/manifests/analytics_cluster/apt.pp
+++ b/modules/profile/manifests/cdh/apt.pp
@@ -1,6 +1,8 @@
-# == Class role::analytics_cluster::apt
+# == Class profile::cdh::apt
 #
-class role::analytics_cluster::apt {
+# Set Cloudera's apt repository to the host.
+#
+class profile::cdh::apt {
     apt::repository { 'thirdparty-cloudera':
         uri        => 'http://apt.wikimedia.org/wikimedia',
         dist       => "${::lsbdistcodename}-wikimedia",
diff --git a/modules/role/manifests/analytics_cluster/hadoop/logstash.pp 
b/modules/profile/manifests/hadoop/logstash.pp
similarity index 85%
rename from modules/role/manifests/analytics_cluster/hadoop/logstash.pp
rename to modules/profile/manifests/hadoop/logstash.pp
index 3f77923..a454109 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/logstash.pp
+++ b/modules/profile/manifests/hadoop/logstash.pp
@@ -1,17 +1,17 @@
-# == Class role::analytics_cluster::hadoop::logstash
+# == Class profile::hadoop::logstash
 # Enables gelf logging to logstash from Hadoop.
 # As of 2016-02, this  is not used.
-class role::analytics_cluster::hadoop::logstash {
-    Class['cdh::hadoop'] -> Class['role::analytics_cluster::hadoop::logstash']
-
+class profile::hadoop::logstash (
+    $gelf_logging_enabled = hiera('profile::hadoop::logstash::enabled'),
+) {
     file { '/usr/local/bin/hadoop-yarn-logging-helper.sh':
-        content => 
template('role/analytics_cluster/hadoop/hadoop-yarn-logging-helper.erb'),
+        content => template('profile/hadoop/hadoop-yarn-logging-helper.erb'),
         mode    => '0744',
     }
 
     $patched_jar_exists_command = '/bin/ls /usr/lib/hadoop-yarn | /bin/grep -E 
 "hadoop-yarn-server-nodemanager.+gelf"'
 
-    if $cdh::hadoop::gelf_logging_enabled {
+    if $gelf_logging_enabled {
         ensure_packages([
             # library dependency
             'libjson-simple-java',
diff --git a/modules/role/manifests/analytics_cluster/java.pp 
b/modules/profile/manifests/java/analytics.pp
similarity index 72%
rename from modules/role/manifests/analytics_cluster/java.pp
rename to modules/profile/manifests/java/analytics.pp
index b538e7d..6c7520c 100644
--- a/modules/role/manifests/analytics_cluster/java.pp
+++ b/modules/profile/manifests/java/analytics.pp
@@ -1,6 +1,11 @@
-# == Class role::analytics_cluster::java
-# Installs the version of Java used for Analytics Cluster.
-class role::analytics_cluster::java {
+# == Class profile::java::analytics
+#
+# Installs Java packages chosen by the Analytics
+# team and used among various projects like Hadoop,
+# Druid, etc. Consistency is essential to allow
+# interoperability among various clusters.
+#
+class profile::java::analytics {
     if os_version('debian >= stretch') {
         require_package('openjdk-8-jdk')
     }
@@ -21,4 +26,4 @@
         line => 'JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF-8"',
         path => '/etc/environment',
     }
-}
+}
\ No newline at end of file
diff --git 
a/modules/role/templates/analytics_cluster/hadoop/hadoop-yarn-logging-helper.erb
 b/modules/profile/templates/hadoop/hadoop-yarn-logging-helper.erb
old mode 100755
new mode 100644
similarity index 100%
rename from 
modules/role/templates/analytics_cluster/hadoop/hadoop-yarn-logging-helper.erb
rename to modules/profile/templates/hadoop/hadoop-yarn-logging-helper.erb
diff --git a/modules/role/manifests/analytics_cluster/database/meta.pp 
b/modules/role/manifests/analytics_cluster/database/meta.pp
index 48fc0d0..3a54d24 100644
--- a/modules/role/manifests/analytics_cluster/database/meta.pp
+++ b/modules/role/manifests/analytics_cluster/database/meta.pp
@@ -4,7 +4,7 @@
 #
 class role::analytics_cluster::database::meta {
     # Some CDH database init scripts need Java to run.
-    require ::role::analytics_cluster::java
+    require ::profile::java::analytics
 
     include ::mariadb::packages_wmf
 
diff --git a/modules/role/manifests/analytics_cluster/druid/common.pp 
b/modules/role/manifests/analytics_cluster/druid/common.pp
index 524d5d9..cc5c83f 100644
--- a/modules/role/manifests/analytics_cluster/druid/common.pp
+++ b/modules/role/manifests/analytics_cluster/druid/common.pp
@@ -12,7 +12,7 @@
 class role::analytics_cluster::druid::common
 {
     # Need Java before Druid is installed.
-    require ::role::analytics_cluster::java
+    require ::profile::java::analytics
 
     $zookeeper_cluster_name = hiera('zookeeper_cluster_name')
     $zookeeper_clusters     = hiera('zookeeper_clusters')
diff --git a/modules/role/manifests/analytics_cluster/hadoop/client.pp 
b/modules/role/manifests/analytics_cluster/hadoop/client.pp
index 1e9120c..54a0e42 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/client.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/client.pp
@@ -6,15 +6,15 @@
     # Include Wikimedia's thirdparty/cloudera apt component
     # as an apt source on all Hadoop hosts.  This is needed
     # to install CDH packages from our apt repo mirror.
-    require ::role::analytics_cluster::apt
+    require ::profile::cdh::apt
 
     # Force apt-get update to run before we try to install packages.
     # CDH Packages are in the thirdparty/cloudera apt component,
-    # and are made available by role::analytics_cluster::apt.
+    # and are made available by profile::cdh::apt.
     Exec['apt-get update'] -> 
Class['::role::analytics_cluster::hadoop::client']
 
     # Need Java before Hadoop is installed.
-    require ::role::analytics_cluster::java
+    require ::profile::java::analytics
 
     $hadoop_var_directory                     = '/var/lib/hadoop'
     $hadoop_name_directory                    = "${hadoop_var_directory}/name"
@@ -82,9 +82,8 @@
             
'yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage' 
=> '99.0',
         },
     }
-    # This will only enable logstash logging if
-    # $cdh::hadoop::gelf_logging_enabled is true.
-    include ::role::analytics_cluster::hadoop::logstash
+
+    include ::profile::hadoop::logstash
 
     # Include ores::base class to get dependencies for using ORES in Hadoop.
     # NOTE: This conditional is temporary, and will be removed once stat1002

-- 
To view, visit https://gerrit.wikimedia.org/r/370187
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: Ic3982c73b80693a81036057cbc510e13a2984c5f
Gerrit-PatchSet: 6
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Elukey <ltosc...@wikimedia.org>
Gerrit-Reviewer: Elukey <ltosc...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: Ottomata <ao...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to