[MediaWiki-commits] [Gerrit] operations/puppet[production]: kafka: create compound roles, one role() call per node defin...
Giuseppe Lavagetto has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/402784 )
Change subject: kafka: create compound roles, one role() call per node
definition
..
kafka: create compound roles, one role() call per node definition
Change-Id: If38f8b18967c883fa7982b5b16751f68e1d69d03
---
A hieradata/role/common/kafka/analytics.yaml
D hieradata/role/common/kafka/analytics/broker.yaml
A hieradata/role/common/kafka/main.yaml
M manifests/site.pp
A modules/role/manifests/kafka/analytics.pp
A modules/role/manifests/kafka/main.pp
6 files changed, 51 insertions(+), 30 deletions(-)
Approvals:
Giuseppe Lavagetto: Looks good to me, approved
jenkins-bot: Verified
diff --git a/hieradata/role/common/kafka/analytics.yaml
b/hieradata/role/common/kafka/analytics.yaml
new file mode 100644
index 000..d714f2f
--- /dev/null
+++ b/hieradata/role/common/kafka/analytics.yaml
@@ -0,0 +1,11 @@
+admin::groups:
+ - analytics-admins
+cluster: analytics_kafka
+nagios_group: analytics_eqiad
+burrow::lagcheck_intervals: 100
+burrow::httpserver_port: 8000
+burrow::consumer_groups:
+ - eventlogging_processor_client_side_00
+ - eventlogging_consumer_mysql_00
+ - eventlogging_consumer_mysql_eventbus_00
+ - eventlogging_consumer_files_00
diff --git a/hieradata/role/common/kafka/analytics/broker.yaml
b/hieradata/role/common/kafka/analytics/broker.yaml
deleted file mode 100644
index 79be700..000
--- a/hieradata/role/common/kafka/analytics/broker.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-admin::groups:
- - analytics-admins
-cluster: analytics_kafka
-nagios_group: analytics_eqiad
diff --git a/hieradata/role/common/kafka/main.yaml
b/hieradata/role/common/kafka/main.yaml
new file mode 100644
index 000..201622d
--- /dev/null
+++ b/hieradata/role/common/kafka/main.yaml
@@ -0,0 +1,5 @@
+cluster: eventbus
+admin::groups:
+ - eventbus-admins
+role::lvs::realserver::pools:
+ eventbus: {}
diff --git a/manifests/site.pp b/manifests/site.pp
index edf431e..15e9db2 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -1066,38 +1066,14 @@
# Analytics Kafka Brokers
node /kafka10(12|13|14|20|22|23)\.eqiad\.wmnet/ {
-# Kafka brokers are routed via IPv6 so that
-# other DCs can address without public IPv4
-# addresses.
-interface::add_ip6_mapped { 'main': }
-
-role(kafka::analytics::broker,
-# Mirror all other Kafka cluster data into the analytics Kafka cluster.
-kafka::analytics::mirror,
-# Mirror main Kafka cluster data to Jumbo Kafka cluster.
-# NOTE: this is only running on the analytics Kafka brokers because
-# of a 0.11 client compatibility issue. Ideally this role would
-# be included on the jumbo brokers instead. But, since we need to
consume
-# from a 0.9 cluster (main), we need to use a non 0.11 MirrorMaker
version,
-# which is not available on the Kafka jumbo brokers, since they are
0.11.
-kafka::jumbo::mirror,
-ipsec)
-
-include ::standard
-include ::base::firewall
+role(kafka::analytics)
}
# Kafka Brokers - main-eqiad and main-codfw Kafka clusters.
# For now, eventlogging-service-eventbus is also colocated
# on these brokers.
node /kafka[12]00[123]\.(eqiad|codfw)\.wmnet/ {
-role(kafka::main::broker,
-# Mirror eqiad.* topics from main-eqiad into main-codfw,
-# or mirror codfw.* topics from main-codfw into main-eqiad.
-kafka::main::mirror,
-eventbus::eventbus)
-
-include ::standard
+role(kafka::main)
}
# kafka-jumbo is a large general purpose Kafka cluster.
diff --git a/modules/role/manifests/kafka/analytics.pp
b/modules/role/manifests/kafka/analytics.pp
new file mode 100644
index 000..d9c5fa9
--- /dev/null
+++ b/modules/role/manifests/kafka/analytics.pp
@@ -0,0 +1,23 @@
+# Compound role for analytics kafka
+class role::kafka::analytics {
+system::role { 'kafka_analytics': }
+# Kafka brokers are routed via IPv6 so that
+# other DCs can address without public IPv4
+# addresses.
+interface::add_ip6_mapped { 'main': }
+
+include ::role::kafka::analytics::broker
+# Mirror all other Kafka cluster data into the analytics Kafka cluster.
+include ::role::kafka::analytics::mirror
+# Mirror main Kafka cluster data to Jumbo Kafka cluster.
+# NOTE: this is only running on the analytics Kafka brokers because
+# of a 0.11 client compatibility issue. Ideally this role would
+# be included on the jumbo brokers instead. But, since we need to consume
+# from a 0.9 cluster (main), we need to use a non 0.11 MirrorMaker version,
+# which is not available on the Kafka jumbo brokers, since they are 0.11.
+include ::role::kafka::jumbo::mirror
+include ::role::ipsec
+
+include ::standard
+include ::base::firewall
+}
diff --git a/modules/role/manifests/kafka/main.pp
[MediaWiki-commits] [Gerrit] operations/puppet[production]: kafka: create compound roles, one role() call per node defin...
Giuseppe Lavagetto has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/402784 )
Change subject: kafka: create compound roles, one role() call per node
definition
..
kafka: create compound roles, one role() call per node definition
Change-Id: If38f8b18967c883fa7982b5b16751f68e1d69d03
---
A hieradata/role/common/kafka/analytics.yaml
D hieradata/role/common/kafka/analytics/broker.yaml
M manifests/site.pp
A modules/role/manifests/kafka/analytics.pp
A modules/role/manifests/kafka/main.pp
5 files changed, 46 insertions(+), 30 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/84/402784/1
diff --git a/hieradata/role/common/kafka/analytics.yaml
b/hieradata/role/common/kafka/analytics.yaml
new file mode 100644
index 000..d714f2f
--- /dev/null
+++ b/hieradata/role/common/kafka/analytics.yaml
@@ -0,0 +1,11 @@
+admin::groups:
+ - analytics-admins
+cluster: analytics_kafka
+nagios_group: analytics_eqiad
+burrow::lagcheck_intervals: 100
+burrow::httpserver_port: 8000
+burrow::consumer_groups:
+ - eventlogging_processor_client_side_00
+ - eventlogging_consumer_mysql_00
+ - eventlogging_consumer_mysql_eventbus_00
+ - eventlogging_consumer_files_00
diff --git a/hieradata/role/common/kafka/analytics/broker.yaml
b/hieradata/role/common/kafka/analytics/broker.yaml
deleted file mode 100644
index 79be700..000
--- a/hieradata/role/common/kafka/analytics/broker.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-admin::groups:
- - analytics-admins
-cluster: analytics_kafka
-nagios_group: analytics_eqiad
diff --git a/manifests/site.pp b/manifests/site.pp
index edf431e..15e9db2 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -1066,38 +1066,14 @@
# Analytics Kafka Brokers
node /kafka10(12|13|14|20|22|23)\.eqiad\.wmnet/ {
-# Kafka brokers are routed via IPv6 so that
-# other DCs can address without public IPv4
-# addresses.
-interface::add_ip6_mapped { 'main': }
-
-role(kafka::analytics::broker,
-# Mirror all other Kafka cluster data into the analytics Kafka cluster.
-kafka::analytics::mirror,
-# Mirror main Kafka cluster data to Jumbo Kafka cluster.
-# NOTE: this is only running on the analytics Kafka brokers because
-# of a 0.11 client compatibility issue. Ideally this role would
-# be included on the jumbo brokers instead. But, since we need to
consume
-# from a 0.9 cluster (main), we need to use a non 0.11 MirrorMaker
version,
-# which is not available on the Kafka jumbo brokers, since they are
0.11.
-kafka::jumbo::mirror,
-ipsec)
-
-include ::standard
-include ::base::firewall
+role(kafka::analytics)
}
# Kafka Brokers - main-eqiad and main-codfw Kafka clusters.
# For now, eventlogging-service-eventbus is also colocated
# on these brokers.
node /kafka[12]00[123]\.(eqiad|codfw)\.wmnet/ {
-role(kafka::main::broker,
-# Mirror eqiad.* topics from main-eqiad into main-codfw,
-# or mirror codfw.* topics from main-codfw into main-eqiad.
-kafka::main::mirror,
-eventbus::eventbus)
-
-include ::standard
+role(kafka::main)
}
# kafka-jumbo is a large general purpose Kafka cluster.
diff --git a/modules/role/manifests/kafka/analytics.pp
b/modules/role/manifests/kafka/analytics.pp
new file mode 100644
index 000..d9c5fa9
--- /dev/null
+++ b/modules/role/manifests/kafka/analytics.pp
@@ -0,0 +1,23 @@
+# Compound role for analytics kafka
+class role::kafka::analytics {
+system::role { 'kafka_analytics': }
+# Kafka brokers are routed via IPv6 so that
+# other DCs can address without public IPv4
+# addresses.
+interface::add_ip6_mapped { 'main': }
+
+include ::role::kafka::analytics::broker
+# Mirror all other Kafka cluster data into the analytics Kafka cluster.
+include ::role::kafka::analytics::mirror
+# Mirror main Kafka cluster data to Jumbo Kafka cluster.
+# NOTE: this is only running on the analytics Kafka brokers because
+# of a 0.11 client compatibility issue. Ideally this role would
+# be included on the jumbo brokers instead. But, since we need to consume
+# from a 0.9 cluster (main), we need to use a non 0.11 MirrorMaker version,
+# which is not available on the Kafka jumbo brokers, since they are 0.11.
+include ::role::kafka::jumbo::mirror
+include ::role::ipsec
+
+include ::standard
+include ::base::firewall
+}
diff --git a/modules/role/manifests/kafka/main.pp
b/modules/role/manifests/kafka/main.pp
new file mode 100644
index 000..8ca7176
--- /dev/null
+++ b/modules/role/manifests/kafka/main.pp
@@ -0,0 +1,10 @@
+# Compound role for the Kafka "main" cluster
+class role::kafka::main {
+include ::role::kafka::main::broker
+# Mirror eqiad.* topics from main-eqiad into main-codfw,
+# or mirror codfw.* topics from main-codfw into
