Ottomata has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/375105 )
Change subject: webperf: Refactor ve.py and add unit tests
......................................................................
webperf: Refactor ve.py and add unit tests
Bug: T110903
Change-Id: I96e81be8d4b1c3dcd9259ae7c4c9ef65b3002b64
---
M modules/webperf/files/ve.py
A modules/webperf/files/ve_expected.txt
A modules/webperf/files/ve_fixture.yaml
3 files changed, 109 insertions(+), 25 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/modules/webperf/files/ve.py b/modules/webperf/files/ve.py
index dbebc3f..7546d87 100644
--- a/modules/webperf/files/ve.py
+++ b/modules/webperf/files/ve.py
@@ -7,37 +7,66 @@
import argparse
import socket
+import unittest
+import yaml
import eventlogging
-ap = argparse.ArgumentParser(description='PerfData StatsD module')
-ap.add_argument('endpoint', help='URI of EventLogging endpoint')
-ap.add_argument('--statsd-host', default='localhost',
- type=socket.gethostbyname)
-ap.add_argument('--statsd-port', default=8125, type=int)
-args = ap.parse_args()
-
-addr = args.statsd_host, args.statsd_port
-sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-
-events = eventlogging.connect(args.endpoint)
-
-for meta in events.filter(schema='Edit'):
+def handle_edit(meta):
event = meta['event']
if event['editor'] == 'visualeditor':
+ action = event['action']
+ if action == 'saveSuccess':
+ metric = 'save'
+ elif action == 'ready':
+ metric = 'load'
+ else:
+ return
+ timing = int(event['action.%s.timing' % action])
+ # Log values between 0.1s an 100s (1.6min) only
+ if timing < 100 or timing > 100000:
+ return
+ return 'VisualEditor.%s:%s|ms' % (metric, timing)
+
+
+if __name__ == '__main__':
+ ap = argparse.ArgumentParser(description='PerfData StatsD module')
+ ap.add_argument('endpoint', help='URI of EventLogging endpoint')
+ ap.add_argument('--statsd-host', default='localhost',
+ type=socket.gethostbyname)
+ ap.add_argument('--statsd-port', default=8125, type=int)
+ args = ap.parse_args()
+
+ addr = args.statsd_host, args.statsd_port
+ sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+
+ events = eventlogging.connect(args.endpoint)
+
+ for meta in events.filter(schema='Edit'):
try:
- action = event['action']
- if action == 'saveSuccess':
- metric = 'save'
- elif action == 'ready':
- metric = 'load'
- else:
- continue
- timing = int(event['action.%s.timing' % action])
- if timing < 100 or timing > 100000:
- continue
- stat = 'VisualEditor.%s:%s|ms' % (metric, timing)
- sock.sendto(stat.encode('utf-8'), addr)
+ stat = handle_edit(meta)
+ if stat is not None:
+ sock.sendto(stat.encode('utf-8'), addr)
except (ValueError, KeyError):
continue
+
+
+# ##### Tests ######
+# To run:
+# python -m unittest ve
+#
+class TestVePerfData(unittest.TestCase):
+ def test_handler(self):
+ with open('ve_fixture.yaml') as fixture_file:
+ fixture = yaml.safe_load(fixture_file)
+ actual = []
+ for meta in fixture:
+ stat = handle_edit(meta)
+ if stat is not None:
+ actual.append(stat)
+ with open('ve_expected.txt') as expected_file:
+ self.assertItemsEqual(
+ actual,
+ expected_file.read().splitlines()
+ )
diff --git a/modules/webperf/files/ve_expected.txt
b/modules/webperf/files/ve_expected.txt
new file mode 100644
index 0000000..f5ed936
--- /dev/null
+++ b/modules/webperf/files/ve_expected.txt
@@ -0,0 +1,11 @@
+VisualEditor.load:1955|ms
+VisualEditor.load:33097|ms
+VisualEditor.load:3077|ms
+VisualEditor.load:2963|ms
+VisualEditor.load:1769|ms
+VisualEditor.save:14379|ms
+VisualEditor.load:1067|ms
+VisualEditor.load:9615|ms
+VisualEditor.load:2391|ms
+VisualEditor.load:16085|ms
+VisualEditor.load:2711|ms
diff --git a/modules/webperf/files/ve_fixture.yaml
b/modules/webperf/files/ve_fixture.yaml
new file mode 100644
index 0000000..a10f256
--- /dev/null
+++ b/modules/webperf/files/ve_fixture.yaml
@@ -0,0 +1,44 @@
+# Random sample from August 2017
+# Anonimised by removing all fields, except:
+# - event.action
+# - event.action.*
+# - event.editor
+# - event.version
+# - schema
+
+- {"event": {"action": "saveSuccess", "editor": "wikitext", "version": 1},
"schema": "Edit"}
+- {"event": {"action": "saveAttempt", "editor": "wikitext", "version": 1},
"schema": "Edit"}
+- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 309305, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 59002, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "ready", "action.ready.timing": 1955, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "saveIntent", "editor": "wikitext", "version": 1},
"schema": "Edit"}
+- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 2903, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "ready", "action.ready.timing": 33097, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 206, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "ready", "action.ready.timing": 3077, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 817, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "ready", "action.ready.timing": 2963, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 5502, "action.abort.type": "switchwithout", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 214, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "ready", "action.ready.timing": 1769, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 3766, "action.abort.type": "switchwithout", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "saveIntent", "action.saveIntent.timing": 32831,
"editor": "visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "saveAttempt", "action.saveAttempt.timing": 1987,
"editor": "visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 3834, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 435, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "saveSuccess", "action.saveSuccess.timing": 14379,
"editor": "visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 128254, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 147, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "saveFailure", "action.saveFailure.type":
"responseUnknown", "editor": "wikitext", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 311, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 619, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "ready", "action.ready.timing": 1067, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 656, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "ready", "action.ready.timing": 9615, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 237, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "ready", "action.ready.timing": 2391, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "init", "action.init.mechanism": "click",
"action.init.timing": 142771, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "abort", "action.abort.mechanism": "navigate",
"action.abort.timing": 6249, "action.abort.type": "nochange", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "ready", "action.ready.timing": 16085, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "init", "action.init.mechanism": "url",
"action.init.timing": 345, "action.init.type": "page", "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
+- {"event": {"action": "ready", "action.ready.timing": 2711, "editor":
"visualeditor", "version": 1}, "schema": "Edit"}
--
To view, visit https://gerrit.wikimedia.org/r/375105
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I96e81be8d4b1c3dcd9259ae7c4c9ef65b3002b64
Gerrit-PatchSet: 3
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Krinkle <[email protected]>
Gerrit-Reviewer: Jforrester <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: Phedenskog <[email protected]>
Gerrit-Reviewer: Volans <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
