[MediaWiki-commits] [Gerrit] operations/puppet[production]: role::an_cluster::hadoop::client: moving to profiles (first ...
Elukey has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/370187 )
Change subject: role::an_cluster::hadoop::client: moving to profiles (first
part)
..
role::an_cluster::hadoop::client: moving to profiles (first part)
The idea of this commit is to slowly start moving to profiles
and remove the indirect hiera lookups in our code (all tagged
with FIXME).
This change also introduces new namespaces under profiles,
namely cdh and hadoop, that in my opinion respect a bit more
our dependencies. It also adds a specific profile for all
the Java Analytics depencencies, included in various clusters
(Hadoop, Druid, etc..).
Bug: T167790
Change-Id: Ic3982c73b80693a81036057cbc510e13a2984c5f
---
A hieradata/eqiad/profile/hadoop/logstash.yaml
R modules/profile/manifests/cdh/apt.pp
R modules/profile/manifests/hadoop/logstash.pp
R modules/profile/manifests/java/analytics.pp
R modules/profile/templates/hadoop/hadoop-yarn-logging-helper.erb
M modules/role/manifests/analytics_cluster/database/meta.pp
M modules/role/manifests/analytics_cluster/druid/common.pp
M modules/role/manifests/analytics_cluster/hadoop/client.pp
8 files changed, 27 insertions(+), 20 deletions(-)
Approvals:
Elukey: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/eqiad/profile/hadoop/logstash.yaml
b/hieradata/eqiad/profile/hadoop/logstash.yaml
new file mode 100644
index 000..962cd8a
--- /dev/null
+++ b/hieradata/eqiad/profile/hadoop/logstash.yaml
@@ -0,0 +1 @@
+profile::hadoop::logstash::enabled: false
\ No newline at end of file
diff --git a/modules/role/manifests/analytics_cluster/apt.pp
b/modules/profile/manifests/cdh/apt.pp
similarity index 67%
rename from modules/role/manifests/analytics_cluster/apt.pp
rename to modules/profile/manifests/cdh/apt.pp
index 514189f..9ba9396 100644
--- a/modules/role/manifests/analytics_cluster/apt.pp
+++ b/modules/profile/manifests/cdh/apt.pp
@@ -1,6 +1,8 @@
-# == Class role::analytics_cluster::apt
+# == Class profile::cdh::apt
#
-class role::analytics_cluster::apt {
+# Set Cloudera's apt repository to the host.
+#
+class profile::cdh::apt {
apt::repository { 'thirdparty-cloudera':
uri=> 'http://apt.wikimedia.org/wikimedia',
dist => "${::lsbdistcodename}-wikimedia",
diff --git a/modules/role/manifests/analytics_cluster/hadoop/logstash.pp
b/modules/profile/manifests/hadoop/logstash.pp
similarity index 85%
rename from modules/role/manifests/analytics_cluster/hadoop/logstash.pp
rename to modules/profile/manifests/hadoop/logstash.pp
index 3f77923..a454109 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/logstash.pp
+++ b/modules/profile/manifests/hadoop/logstash.pp
@@ -1,17 +1,17 @@
-# == Class role::analytics_cluster::hadoop::logstash
+# == Class profile::hadoop::logstash
# Enables gelf logging to logstash from Hadoop.
# As of 2016-02, this is not used.
-class role::analytics_cluster::hadoop::logstash {
-Class['cdh::hadoop'] -> Class['role::analytics_cluster::hadoop::logstash']
-
+class profile::hadoop::logstash (
+$gelf_logging_enabled = hiera('profile::hadoop::logstash::enabled'),
+) {
file { '/usr/local/bin/hadoop-yarn-logging-helper.sh':
-content =>
template('role/analytics_cluster/hadoop/hadoop-yarn-logging-helper.erb'),
+content => template('profile/hadoop/hadoop-yarn-logging-helper.erb'),
mode=> '0744',
}
$patched_jar_exists_command = '/bin/ls /usr/lib/hadoop-yarn | /bin/grep -E
"hadoop-yarn-server-nodemanager.+gelf"'
-if $cdh::hadoop::gelf_logging_enabled {
+if $gelf_logging_enabled {
ensure_packages([
# library dependency
'libjson-simple-java',
diff --git a/modules/role/manifests/analytics_cluster/java.pp
b/modules/profile/manifests/java/analytics.pp
similarity index 72%
rename from modules/role/manifests/analytics_cluster/java.pp
rename to modules/profile/manifests/java/analytics.pp
index b538e7d..6c7520c 100644
--- a/modules/role/manifests/analytics_cluster/java.pp
+++ b/modules/profile/manifests/java/analytics.pp
@@ -1,6 +1,11 @@
-# == Class role::analytics_cluster::java
-# Installs the version of Java used for Analytics Cluster.
-class role::analytics_cluster::java {
+# == Class profile::java::analytics
+#
+# Installs Java packages chosen by the Analytics
+# team and used among various projects like Hadoop,
+# Druid, etc. Consistency is essential to allow
+# interoperability among various clusters.
+#
+class profile::java::analytics {
if os_version('debian >= stretch') {
require_package('openjdk-8-jdk')
}
@@ -21,4 +26,4 @@
line => 'JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF-8"',
path => '/etc/environment',
}
-}
+}
\ No newline at end of file
diff --git
a/modules/role/templates/analytics_cluster/hadoop/hadoop-yarn-logging-helper.erb
b/modules/profile/templates/hadoop/hadoop-yarn-log
[MediaWiki-commits] [Gerrit] operations/puppet[production]: role::an_cluster::hadoop::client: moving to profiles (first ...
Elukey has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/370187 )
Change subject: role::an_cluster::hadoop::client: moving to profiles (first
part)
..
role::an_cluster::hadoop::client: moving to profiles (first part)
The idea of this commit is to slowly start moving to profiles
and remove indirect hiera lookups in the analytics code.
Bug: T167790
Change-Id: Ic3982c73b80693a81036057cbc510e13a2984c5f
---
A hieradata/eqiad/profile/hadoop/logstash.yaml
R modules/profile/manifests/cdh/apt.pp
R modules/profile/manifests/cdh/java.pp
R modules/profile/manifests/hadoop/logstash.pp
R modules/profile/templates/hadoop/hadoop-yarn-logging-helper.erb
M modules/role/manifests/analytics_cluster/hadoop/client.pp
6 files changed, 18 insertions(+), 16 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/87/370187/1
diff --git a/hieradata/eqiad/profile/hadoop/logstash.yaml
b/hieradata/eqiad/profile/hadoop/logstash.yaml
new file mode 100644
index 000..962cd8a
--- /dev/null
+++ b/hieradata/eqiad/profile/hadoop/logstash.yaml
@@ -0,0 +1 @@
+profile::hadoop::logstash::enabled: false
\ No newline at end of file
diff --git a/modules/role/manifests/analytics_cluster/apt.pp
b/modules/profile/manifests/cdh/apt.pp
similarity index 67%
rename from modules/role/manifests/analytics_cluster/apt.pp
rename to modules/profile/manifests/cdh/apt.pp
index 514189f..f4e8460 100644
--- a/modules/role/manifests/analytics_cluster/apt.pp
+++ b/modules/profile/manifests/cdh/apt.pp
@@ -1,6 +1,8 @@
-# == Class role::analytics_cluster::apt
+# == Class profile::cdh::apt
#
-class role::analytics_cluster::apt {
+# Set Cloudera's apt repository to the host.
+#
+class profile::cdh::apt {
apt::repository { 'thirdparty-cloudera':
uri=> 'http://apt.wikimedia.org/wikimedia',
dist => "${::lsbdistcodename}-wikimedia",
diff --git a/modules/role/manifests/analytics_cluster/java.pp
b/modules/profile/manifests/cdh/java.pp
similarity index 90%
rename from modules/role/manifests/analytics_cluster/java.pp
rename to modules/profile/manifests/cdh/java.pp
index b538e7d..900e3eb 100644
--- a/modules/role/manifests/analytics_cluster/java.pp
+++ b/modules/profile/manifests/cdh/java.pp
@@ -1,6 +1,6 @@
-# == Class role::analytics_cluster::java
+# == Class profile::cdh::java
# Installs the version of Java used for Analytics Cluster.
-class role::analytics_cluster::java {
+class profile::cdh::java {
if os_version('debian >= stretch') {
require_package('openjdk-8-jdk')
}
@@ -21,4 +21,4 @@
line => 'JAVA_TOOL_OPTIONS="-Dfile.encoding=UTF-8"',
path => '/etc/environment',
}
-}
+}
\ No newline at end of file
diff --git a/modules/role/manifests/analytics_cluster/hadoop/logstash.pp
b/modules/profile/manifests/hadoop/logstash.pp
similarity index 85%
rename from modules/role/manifests/analytics_cluster/hadoop/logstash.pp
rename to modules/profile/manifests/hadoop/logstash.pp
index 3f77923..123acc2 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/logstash.pp
+++ b/modules/profile/manifests/hadoop/logstash.pp
@@ -1,17 +1,17 @@
-# == Class role::analytics_cluster::hadoop::logstash
+# == Class profile::hadoop::logstash
# Enables gelf logging to logstash from Hadoop.
# As of 2016-02, this is not used.
-class role::analytics_cluster::hadoop::logstash {
-Class['cdh::hadoop'] -> Class['role::analytics_cluster::hadoop::logstash']
-
+class profile::hadoop::logstash { (
+gelf_logging_enabled = hiera('profile::hadoop::logstash::enabled'),
+) {
file { '/usr/local/bin/hadoop-yarn-logging-helper.sh':
-content =>
template('role/analytics_cluster/hadoop/hadoop-yarn-logging-helper.erb'),
+content => template('profile/hadoop/hadoop-yarn-logging-helper.erb'),
mode=> '0744',
}
$patched_jar_exists_command = '/bin/ls /usr/lib/hadoop-yarn | /bin/grep -E
"hadoop-yarn-server-nodemanager.+gelf"'
-if $cdh::hadoop::gelf_logging_enabled {
+if $gelf_logging_enabled {
ensure_packages([
# library dependency
'libjson-simple-java',
diff --git
a/modules/role/templates/analytics_cluster/hadoop/hadoop-yarn-logging-helper.erb
b/modules/profile/templates/hadoop/hadoop-yarn-logging-helper.erb
old mode 100755
new mode 100644
similarity index 100%
rename from
modules/role/templates/analytics_cluster/hadoop/hadoop-yarn-logging-helper.erb
rename to modules/profile/templates/hadoop/hadoop-yarn-logging-helper.erb
diff --git a/modules/role/manifests/analytics_cluster/hadoop/client.pp
b/modules/role/manifests/analytics_cluster/hadoop/client.pp
index 1e9120c..b7ff1fb 100644
--- a/modules/role/manifests/analytics_cluster/hadoop/client.pp
+++ b/modules/role/manifests/analytics_cluster/hadoop/client.pp
@@ -6,7 +6,7 @@
# Include Wikimedia's thirdparty/cloudera apt component
# as an apt source on al
