Giuseppe Lavagetto has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/377209 )
Change subject: eventstreams: convert to role/profile ...................................................................... eventstreams: convert to role/profile Change-Id: I94dad66103ce344a04b85a581203ecc5540052d0 --- M hieradata/labs/deployment-prep/common.yaml M hieradata/role/common/scb.yaml D modules/eventstreams/manifests/init.pp A modules/profile/manifests/eventstreams.pp M modules/role/manifests/eventstreams.pp 5 files changed, 84 insertions(+), 128 deletions(-) Approvals: Giuseppe Lavagetto: Looks good to me, approved jenkins-bot: Verified diff --git a/hieradata/labs/deployment-prep/common.yaml b/hieradata/labs/deployment-prep/common.yaml index 7a35d08..a1fbef7 100644 --- a/hieradata/labs/deployment-prep/common.yaml +++ b/hieradata/labs/deployment-prep/common.yaml @@ -341,3 +341,14 @@ prometheus_nodes: - deployment-prometheus01.deployment-prep.eqiad.wmflabs profile::recommendation_api::wdqs_uri: http://wdqs-test.wmflabs.org + +# Eventstreams config +profile::eventstreams::kafka_cluster_name: main +profile::eventstreams::streams: + test: + topics: ["%{::site}.test.event"] + revision-create: + topics: ["%{::site}.mediawiki.revision-create"] + recentchange: + topics: ["%{::site}.mediawiki.recentchange"] +profile::eventstreams::rdkafka_config: {} diff --git a/hieradata/role/common/scb.yaml b/hieradata/role/common/scb.yaml index e6134b5..df1763b 100644 --- a/hieradata/role/common/scb.yaml +++ b/hieradata/role/common/scb.yaml @@ -56,9 +56,12 @@ # The password is in the private store, this is here for completeness # profile::ores::web::redis_password: nothing + +### BEGIN EVENTSTREAMS ### +profile::eventstreams::kafka_cluster_name: analytics # Stream configuration for Public EventStreams service # Maps stream route names to composite Kafka topics. -role::eventstreams::streams: +profile::eventstreams::streams: recentchange: description: "Mediawiki RecentChanges feed. Schema: https://github.com/wikimedia/mediawiki-event-schemas/tree/master/jsonschema/mediawiki/recentchange"; topics: @@ -76,7 +79,7 @@ - codfw.test.event # rdkafka config for Public EventStreams service. -role::eventstreams::rdkafka_config: +profile::eventstreams::rdkafka_config: # Send rdkafka stats to statsd once per minute. statistics.interval.ms: 60000 # Specify Kafka API version as workaround for Brokers < 0.10 @@ -84,6 +87,7 @@ # This will not be necessary when the target Kafka cluster will be running 0.10 # librdkafka 0.9.4.x default for api.version.request is false, no need to set it broker.version.fallback: '0.9.0.1' +### END EVENTSTREAMS ### changeprop::restbase_uri: http://restbase-async.discovery.wmnet:7231 profile::nutcracker::memcached_pools: {} diff --git a/modules/eventstreams/manifests/init.pp b/modules/eventstreams/manifests/init.pp deleted file mode 100644 index 011dec8..0000000 --- a/modules/eventstreams/manifests/init.pp +++ /dev/null @@ -1,57 +0,0 @@ -# == Class: eventstreams -# -# === Parameters -# -# [*broker_list*] -# Comma-separated list of Kafka broker URIs -# -# [*streams*] -# Hash of stream route config and their composite topics. E.g. -# -# streamName1: -# topics: [topicA, topicB] -# streamName2: -# topics: [topicC, topicD] -# -# [*port*] -# Default: 8092 -# -# [*log_level*] -# Log level for service logger. Default: info -# -# [*rdkafka_config*] -# Extra librdkafka configuration to provide to node-rdkafka. Default: {} -# -class eventstreams( - $broker_list, - $streams, - $port = 8092, - $log_level = 'info', - $rdkafka_config = {}, -) { - service::packages { 'eventstreams': - pkgs => ['librdkafka++1', 'librdkafka1'], - } - - service::node { 'eventstreams': - enable => true, - port => $port, - has_spec => false, # TODO: figure out how to monitor stream with spec x-amples - deployment => 'scap3', - deployment_config => true, - deployment_vars => { - log_level => $log_level, - site => $::site, - broker_list => $broker_list, - rdkafka_config => $rdkafka_config, - streams => $streams, - }, - auto_refresh => false, - init_restart => false, - environment => { - 'UV_THREADPOOL_SIZE' => 128, - }, - require => Service::Packages['eventstreams'], - } - -} diff --git a/modules/profile/manifests/eventstreams.pp b/modules/profile/manifests/eventstreams.pp new file mode 100644 index 0000000..d340984 --- /dev/null +++ b/modules/profile/manifests/eventstreams.pp @@ -0,0 +1,66 @@ +# == Class profile::eventstreams +# +# Profile that installs EventStreams HTTP service. +# This class includes the ::eventstreams role, and configures +# it to consume only specific topics from a specific Kafka cluster. +# +# NOTE: eventstreams is configured to use the 'analytics-eqiad' Kafka cluster +# in both eqiad and codfw. This means that codfw eventstreams will be doing +# cross DC consumption. Since we don't have a beefy non critical 'aggregate/analytics' +# Kafka cluster in codfw (yet), we don't have a good Kafka cluster that can back +# eventstreams in codfw. We could back it with the main-codfw cluster, but there +# are issues with doing that: +# +# 1. main Kafka clusters are for critical prod services, and we don't want some unknown +# bug/exploit taking down the main clusters. This service 'exposes' Kafka to the internet. +# +# 2. change-prop requires that topics have only one partition, which means we can't spread +# consumer load for a given topic across multiple brokers. change-prop uses the main +# Kafka clusters. In the analytics/aggregate cluster, we can add partitions to the topics. +# +# In the future, we would like to rename analytics-eqiad to something more appropriate, perhaps +# aggregate-eqiad, and also set up an aggregate-codfw Kafka cluster. +# +# == Parameters +# [*kafka_cluster_name*] +# Name of the kafka cluster +# +# [*streams*] +# Streams to follow +# +# [*rdkafka_config*] +# Additional configuration for the kafka library +# +# filtertags: labs-project-deployment-prep +class profile::eventstreams( + $kafka_cluster_name = hiera('profile::eventstreams::kafka_cluster_name'), + $streams = hiera('profile::eventstreams::streams'), + $rdkafka_config = hiera('profile::eventstreams::rdkafka_config') +) { + $kafka_config = kafka_config($kafka_cluster_name) + $broker_list = $kafka_config['brokers']['string'] + service::packages { 'eventstreams': + pkgs => ['librdkafka++1', 'librdkafka1'], + } + + service::node { 'eventstreams': + enable => true, + port => 8092, + has_spec => false, # TODO: figure out how to monitor stream with spec x-amples + deployment => 'scap3', + deployment_config => true, + deployment_vars => { + log_level => 'info', + site => $::site, + broker_list => $broker_list, + rdkafka_config => $rdkafka_config, + streams => $streams, + }, + auto_refresh => false, + init_restart => false, + environment => { + 'UV_THREADPOOL_SIZE' => 128, + }, + require => Service::Packages['eventstreams'], + } +} diff --git a/modules/role/manifests/eventstreams.pp b/modules/role/manifests/eventstreams.pp index 6972948..3bb4d58 100644 --- a/modules/role/manifests/eventstreams.pp +++ b/modules/role/manifests/eventstreams.pp @@ -4,77 +4,9 @@ # This class includes the ::eventstreams role, and configures # it to consume only specific topics from a specific Kafka cluster. # -# NOTE: eventstreams is configured to use the 'analytics-eqiad' Kafka cluster -# in both eqiad and codfw. This means that codfw eventstreams will be doing -# cross DC consumption. Since we don't have a beefy non critical 'aggregate/analytics' -# Kafka cluster in codfw (yet), we don't have a good Kafka cluster that can back -# eventstreams in codfw. We could back it with the main-codfw cluster, but there -# are issues with doing that: -# -# 1. main Kafka clusters are for critical prod services, and we don't want some unknown -# bug/exploit taking down the main clusters. This service 'exposes' Kafka to the internet. -# -# 2. change-prop requires that topics have only one partition, which means we can't spread -# consumer load for a given topic across multiple brokers. change-prop uses the main -# Kafka clusters. In the analytics/aggregate cluster, we can add partitions to the topics. -# -# In the future, we would like to rename analytics-eqiad to something more appropriate, perhaps -# aggregate-eqiad, and also set up an aggregate-codfw Kafka cluster. -# -# == Hiera Variables -# [*role::eventstreams::kafka_cluster_name*] -# Default: 'analytics' in production, 'main' in labs. -# -# [*role::eventstreams::port*] -# Default: 8092 -# -# filtertags: labs-project-deployment-prep -# [*role::eventstreams::log_level*] -# Default: info -# -# [*role::eventstreams::streams*] -# Default: test and revision-create. -# -# [*role::eventstreams::rdkafka_config*] -# Default: {} -# class role::eventstreams { system::role { 'eventstreams': description => 'Exposes configured event streams from Kafka to public internet via HTTP SSE', } - - # Default to analytics-eqiad in production, for both eqiad and codfw eventstreams. - $default_kafka_cluster_name = $::realm ? { - 'production' => 'analytics', - 'labs' => 'main' - } - - $kafka_cluster_name = hiera('role::eventstreams::kafka_cluster_name', $default_kafka_cluster_name) - $kafka_config = kafka_config($kafka_cluster_name) - - $port = hiera('role::eventstreams::port', 8092) - $log_level = hiera('role::eventstreams::log_level', 'info') - - $streams = hiera('role::eventstreams::streams', { - 'test' => { - 'topics' => ["${::site}.test.event"] - }, - 'revision-create' => { - 'topics' => ["${::site}.mediawiki.revision-create"] - }, - 'recentchange' => { - 'topics' => ["${::site}.mediawiki.recentchange"] - }, - }) - - # Any extra librdkafka configuration - $rdkafka_config = hiera('role::eventstreams::rdkafka_config', {}) - - class { '::eventstreams': - port => $port, - log_level => $log_level, - broker_list => $kafka_config['brokers']['string'], - streams => $streams, - rdkafka_config => $rdkafka_config, - } + include ::profile::eventstreams } -- To view, visit https://gerrit.wikimedia.org/r/377209 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I94dad66103ce344a04b85a581203ecc5540052d0 Gerrit-PatchSet: 3 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
