Ottomata has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/364881 )
Change subject: Set up separate EventLogging Analytics MySQL Consumer process for eventbus events ...................................................................... Set up separate EventLogging Analytics MySQL Consumer process for eventbus events This caused a bug with the main eventlogging mysql consumer. It was flapping every 5 minutes and dying, because inserts into the mediawiki_page_create_1 table were failing, because https://gerrit.wikimedia.org/r/#/c/362321/ added a field to the schema of the same version. We need to start properly versioning eventbus schemas. Change-Id: I2df1d8eae141e7b51e6903aa906e613817fc0659 --- M modules/role/manifests/eventlogging/analytics/mysql.pp 1 file changed, 30 insertions(+), 11 deletions(-) Approvals: Ottomata: Looks good to me, approved jenkins-bot: Verified diff --git a/modules/role/manifests/eventlogging/analytics/mysql.pp b/modules/role/manifests/eventlogging/analytics/mysql.pp index 5b4a8ce..b7c95b6 100644 --- a/modules/role/manifests/eventlogging/analytics/mysql.pp +++ b/modules/role/manifests/eventlogging/analytics/mysql.pp @@ -56,15 +56,6 @@ $topics_to_consume = [ # Valid eventlogging analytics events are all in this one topic. 'eventlogging-valid-mixed', - # Various mediawiki events (via EventBus) - 'eqiad.mediawiki.page-create', - 'codfw.mediawiki.page-create', - 'eqiad.mediawiki.page-move', - 'codfw.mediawiki.page-move', - 'eqiad.mediawiki.page-delete', - 'codfw.mediawiki.page-delete', - 'eqiad.mediawiki.page-undelete', - 'codfw.mediawiki.page-undelete', ] $topics_string = join($topics_to_consume, ',') $kafka_consumer_uri = "${kafka_consumer_scheme}/${kafka_brokers_string}?topics=${topics_string}" @@ -82,7 +73,36 @@ # Kafka consumer group for this consumer is mysql-m4-master eventlogging::service::consumer { $mysql_consumers: # auto commit offsets to kafka more often for mysql consumer - input => "${filter_scheme}${kafka_consumer_uri}&auto_commit_interval_ms=1000${$kafka_api_version_param}${filter_function}", + input => "${filter_scheme}${kafka_consumer_uri}&auto_commit_interval_ms=1000${kafka_api_version_param}${filter_function}", + output => "mysql://${mysql_user}:${mysql_pass}@${mysql_db}?charset=utf8&statsd_host=${statsd_host}&replace=True", + sid => $kafka_consumer_group, + # Restrict permissions on this config file since it contains a password. + owner => 'root', + group => 'eventlogging', + mode => '0640', + } + + + $eventbus_topics_to_consume = [ + # Various mediawiki events (via EventBus) + 'eqiad.mediawiki.page-create', + 'codfw.mediawiki.page-create', + 'eqiad.mediawiki.page-move', + 'codfw.mediawiki.page-move', + 'eqiad.mediawiki.page-delete', + 'codfw.mediawiki.page-delete', + 'eqiad.mediawiki.page-undelete', + 'codfw.mediawiki.page-undelete', + ] + $eventbus_topics_string = join($topics_to_consume, ',') + $kafka_consumer_uri_eventbus = "${kafka_consumer_scheme}/${kafka_brokers_string}?topics=${eventbus_topics_string}" + + # Use a separate mysql consumer process to insert eventbus events. + # The schemas for these types of events are managed differently, and we don't + # want bugs in one to affect the other. + eventlogging::service::consumer { 'mysql-eventbus': + # auto commit offsets to kafka more often for mysql consumer + input => "${kafka_consumer_uri_eventbus}&auto_commit_interval_ms=1000${kafka_api_version_param}", output => "mysql://${mysql_user}:${mysql_pass}@${mysql_db}?charset=utf8&statsd_host=${statsd_host}&replace=True", # Load and cache local (EventBus) schemas so those events can be inserted into MySQL too. # This will require a restart of the consumer process(es) when there are any new schemas. @@ -92,6 +112,5 @@ owner => 'root', group => 'eventlogging', mode => '0640', - } } -- To view, visit https://gerrit.wikimedia.org/r/364881 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I2df1d8eae141e7b51e6903aa906e613817fc0659 Gerrit-PatchSet: 2 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ottomata <ao...@wikimedia.org> Gerrit-Reviewer: Ottomata <ao...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
