Dzahn has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/376146 )
Change subject: webperf: Decom webperf::ve service
......................................................................
webperf: Decom webperf::ve service
Bug: T175083
Change-Id: I76bf0e43d700120551f79013a2a6dd7394ab63be
---
M modules/role/manifests/webperf.pp
D modules/webperf/files/ve.py
D modules/webperf/files/ve_expected.txt
D modules/webperf/files/ve_fixture.yaml
D modules/webperf/manifests/ve.pp
D modules/webperf/templates/ve.systemd.erb
6 files changed, 0 insertions(+), 216 deletions(-)
Approvals:
jenkins-bot: Verified
Dzahn: Looks good to me, approved
diff --git a/modules/role/manifests/webperf.pp
b/modules/role/manifests/webperf.pp
index ab0d806..80b8388 100644
--- a/modules/role/manifests/webperf.pp
+++ b/modules/role/manifests/webperf.pp
@@ -31,12 +31,4 @@
statsd_host => $statsd_host,
statsd_port => $statsd_port,
}
-
- # Report VisualEditor performance measurements to Graphite.
- # See <https://meta.wikimedia.org/wiki/Schema:Edit>
- class { '::webperf::ve':
- kafka_brokers => $kafka_brokers,
- statsd_host => $statsd_host,
- statsd_port => $statsd_port,
- }
}
diff --git a/modules/webperf/files/ve.py b/modules/webperf/files/ve.py
deleted file mode 100644
index c8318e5..0000000
--- a/modules/webperf/files/ve.py
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-import sys
-reload(sys)
-sys.setdefaultencoding("utf-8")
-
-import argparse
-import json
-import socket
-import unittest
-import yaml
-
-from kafka import KafkaConsumer
-
-
-def handle_edit(meta):
- event = meta['event']
- if event['editor'] == 'visualeditor':
- action = event['action']
- if action == 'saveSuccess':
- metric = 'save'
- elif action == 'ready':
- metric = 'load'
- else:
- return
- timing = int(event['action.%s.timing' % action])
- # Log values between 0.1s an 100s (1.6min) only
- if timing < 100 or timing > 100000:
- return
- return 'VisualEditor.%s:%s|ms' % (metric, timing)
-
-
-if __name__ == '__main__':
- ap = argparse.ArgumentParser(description='Send VisualEditor PerfData to
StatsD')
- ap.add_argument('--brokers', required=True,
- help='Comma-separated list of kafka brokers')
- ap.add_argument('--consumer-group', required=True,
- help='Consumer group to register with Kafka')
- ap.add_argument('--statsd-host', default='localhost',
- type=socket.gethostbyname)
- ap.add_argument('--statsd-port', default=8125, type=int)
- args = ap.parse_args()
-
- addr = args.statsd_host, args.statsd_port
- sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-
- kafka_bootstrap_servers = tuple(args.brokers.split(','))
- kafka_topic = 'eventlogging_Edit'
- kafka_consumer_timeout_seconds = 600
- consumer = KafkaConsumer(
- kafka_topic,
- bootstrap_servers=kafka_bootstrap_servers,
- group_id=args.consumer_group,
- auto_offset_reset='latest',
- enable_auto_commit=False,
- consumer_timeout_ms=kafka_consumer_timeout_seconds * 1000
- )
-
- try:
- for message in consumer:
- meta = json.loads(message.value)
- try:
- stat = handle_edit(meta)
- if stat is not None:
- sock.sendto(stat.encode('utf-8'), addr)
- except (ValueError, KeyError):
- continue
- # If we reach this line, consumer_timeout_ms elapsed without events
- raise RuntimeError('No messages received in %d seconds.' %
kafka_consumer_timeout_seconds)
- finally:
- consumer.close()
-
-
-# ##### Tests ######
-# To run:
-# python -m unittest ve
-#
-class TestVePerfData(unittest.TestCase):
- def test_handler(self):
- with open('ve_fixture.yaml') as fixture_file:
- fixture = yaml.safe_load(fixture_file)
- actual = []
- for meta in fixture:
- stat = handle_edit(meta)
- if stat is not None:
- actual.append(stat)
- with open('ve_expected.txt') as expected_file:
- self.assertItemsEqual(
- actual,
- expected_file.read().splitlines()
- )
diff --git a/modules/webperf/files/ve_expected.txt
b/modules/webperf/files/ve_expected.txt
deleted file mode 100644
index f5ed936..0000000
--- a/modules/webperf/files/ve_expected.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-VisualEditor.load:1955|ms
-VisualEditor.load:33097|ms
-VisualEditor.load:3077|ms
-VisualEditor.load:2963|ms
-VisualEditor.load:1769|ms
-VisualEditor.save:14379|ms
-VisualEditor.load:1067|ms
-VisualEditor.load:9615|ms
-VisualEditor.load:2391|ms
-VisualEditor.load:16085|ms
-VisualEditor.load:2711|ms
diff --git a/modules/webperf/files/ve_fixture.yaml
b/modules/webperf/files/ve_fixture.yaml
deleted file mode 100644
index a10f256..0000000
--- a/modules/webperf/files/ve_fixture.yaml
+++ /dev/null
@@ -1,44 +0,0 @@
-# Random sample from August 2017
-# Anonimised by removing all fields, except:
-# - event.action
-# - event.action.*
-# - event.editor
-# - event.version
-# - schema
-
-- {"event": {"action": "saveSuccess", "editor": "wikitext", "version": 1},
"schema": "Edit"}
-- {"event": {"action": "saveAttempt", "editor": "wikitext", "version": 1},
"schema": "Edit"}
-- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 309305, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 59002, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "ready", "action.ready.timing": 1955, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "saveIntent", "editor": "wikitext", "version": 1},
"schema": "Edit"}
-- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 2903, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "ready", "action.ready.timing": 33097, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 206, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "ready", "action.ready.timing": 3077, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 817, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "ready", "action.ready.timing": 2963, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 5502, "action.abort.type": "switchwithout", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 214, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "ready", "action.ready.timing": 1769, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 3766, "action.abort.type": "switchwithout", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "saveIntent", "action.saveIntent.timing": 32831,
"editor": "visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "saveAttempt", "action.saveAttempt.timing": 1987,
"editor": "visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 3834, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 435, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "saveSuccess", "action.saveSuccess.timing": 14379,
"editor": "visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 128254, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 147, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "saveFailure", "action.saveFailure.type":
"responseUnknown", "editor": "wikitext", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 311, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 619, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "ready", "action.ready.timing": 1067, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 656, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "ready", "action.ready.timing": 9615, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 237, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "ready", "action.ready.timing": 2391, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "init", "action.init.mechanism": "click",
"action.init.timing": 142771, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 6249, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "ready", "action.ready.timing": 16085, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 345, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
-- {"event": {"action": "ready", "action.ready.timing": 2711, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
diff --git a/modules/webperf/manifests/ve.pp b/modules/webperf/manifests/ve.pp
deleted file mode 100644
index d5f5c7c..0000000
--- a/modules/webperf/manifests/ve.pp
+++ /dev/null
@@ -1,45 +0,0 @@
-# == Class: webperf::ve
-#
-# Captures VisualEditor timing data and sends it to StatsD.
-# See <https://meta.wikimedia.org/wiki/Schema:Edit>.
-#
-# === Parameters
-#
-# [*kafka_brokers*]
-# String of comma separated Kafka bootstrap brokers.
-#
-# [*statsd_host*]
-# Write stats to this StatsD instance. Default: '127.0.0.1'.
-#
-# [*statsd_port*]
-# Write stats to this StatsD instance. Default: 8125.
-#
-class webperf::ve(
- $kafka_brokers,
- $statsd_host = '127.0.0.1',
- $statsd_port = 8125,
-) {
- include ::webperf
-
- require_package('python-kafka')
- require_package('python-yaml')
-
- file { '/srv/webperf/ve.py':
- source => 'puppet:///modules/webperf/ve.py',
- owner => 'www-data',
- group => 'www-data',
- mode => '0755',
- notify => Service['ve'],
- }
-
- file { '/lib/systemd/system/ve.service':
- # uses $statsd_host, $statsd_port, $kafka_brokers
- content => template('webperf/ve.systemd.erb'),
- notify => Service['ve'],
- }
-
- service { 've':
- ensure => running,
- provider => systemd,
- }
-}
diff --git a/modules/webperf/templates/ve.systemd.erb
b/modules/webperf/templates/ve.systemd.erb
deleted file mode 100644
index 980b477..0000000
--- a/modules/webperf/templates/ve.systemd.erb
+++ /dev/null
@@ -1,16 +0,0 @@
-[Unit]
-Description=ve
-After=network-online.target
-
-[Service]
-WorkingDirectory=/srv/webperf
-ExecStart=/usr/bin/python /srv/webperf/ve.py \
- --brokers <%= @kafka_brokers %> \
- --consumer-group webperf_ve \
- --statsd-host <%= @statsd_host %> \
- --statsd-port <%= @statsd_port %>
-User=nobody
-Restart=always
-
-[Install]
-WantedBy=multi-user.target
--
To view, visit https://gerrit.wikimedia.org/r/376146
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I76bf0e43d700120551f79013a2a6dd7394ab63be
Gerrit-PatchSet: 2
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Krinkle <krinklem...@gmail.com>
Gerrit-Reviewer: Dzahn <dz...@wikimedia.org>
Gerrit-Reviewer: Gilles <gdu...@wikimedia.org>
Gerrit-Reviewer: Giuseppe Lavagetto <glavage...@wikimedia.org>
Gerrit-Reviewer: Jforrester <jforres...@wikimedia.org>
Gerrit-Reviewer: Ottomata <ao...@wikimedia.org>
Gerrit-Reviewer: Volans <rcocci...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
