Ori.livneh has uploaded a new change for review.
https://gerrit.wikimedia.org/r/73891
Change subject: Import mwerrors ganglia metric module
......................................................................
Import mwerrors ganglia metric module
This was previously in modules/eventlogging in operations/puppet, but is being
spun off into its own thing.
Change-Id: Ic4990efa56b9af31f6416211b66c311fe9f4ffbb
---
M errproc.py
A metric_module/mwerrors.py
A metric_module/mwerrors.pyconf.sample
3 files changed, 179 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/tools/fluoride
refs/changes/91/73891/1
diff --git a/errproc.py b/errproc.py
index 95cec35..1ba6dfa 100644
--- a/errproc.py
+++ b/errproc.py
@@ -70,12 +70,15 @@
return exception
-pattern = (r'\[(?P<timestamp>[^\]]+)\] Fatal error: (?P<message>.*) at '
- r'(?P<file>\/[\S]+) on line (?P<line>\d+)')
+fatal_pattern = (r'\[(?P<timestamp>[^\]]+)\] Fatal error: (?P<message>.*) at '
+ r'(?P<file>\/[\S]+) on line (?P<line>\d+)')
+"""
+e.g.:
for line in open('fatal.log'):
if line.startswith('[') and not re.match(pattern, line):
print line
+"""
def iter_exceptions(buffer):
diff --git a/metric_module/mwerrors.py b/metric_module/mwerrors.py
new file mode 100755
index 0000000..1871722
--- /dev/null
+++ b/metric_module/mwerrors.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+ Gmond metric-gathering module for MediaWiki fatals and exceptions
+
+ Reads fatals / exceptions from a ZeroMQ publisher. MediaWiki logs to a file
+ or a UDP socket, so for this to work you will also need a UDP-to-ZMQ router.
+ See 'udp2zmq' in EventLogging.
+
+ When invoked by itself, runs a self-test.
+
+ Usage: mwerrors.py tcp://HOST:PORT
+
+ Written by Ori Livneh <[email protected]>
+
+"""
+import sys
+reload(sys)
+sys.setdefaultencoding('utf8')
+
+import errno
+import threading
+import time
+
+import zmq
+
+
+patterns = (
+ # Substring to match # Metric # Metric title
+ ('Fatal error: Out of memory', 'oom', 'Out-of-memory
fatals'),
+ ('Fatal error: Maximum execution time', 'timelimit', 'Time limit
fatals'),
+ ('Fatal error:', 'fatal', 'Miscellaneous
fatals'),
+ ('Exception from', 'exception', 'Exceptions'),
+ ('Catchable fatal error', 'catchable', 'Catchable fatals'),
+ ('DatabaseBase->reportQueryError', 'query', 'Query errors'),
+)
+
+
+def count_errors(counter, endpoint):
+ """Count error types in error stream."""
+ ctx = zmq.Context.instance()
+ sock = ctx.socket(zmq.SUB)
+ sock.connect(endpoint)
+ sock.setsockopt(zmq.SUBSCRIBE, b'')
+
+ while 1:
+ try:
+ line = sock.recv()
+ for pattern, name, description in patterns:
+ if pattern in line:
+ counter[name] += 1
+ break
+ except zmq.ZMQError as e:
+ # Calls interrupted by EINTR should be re-tried.
+ if e.errno == errno.EINTR:
+ continue
+ raise
+
+
+def metric_init(params):
+ """
+ Initialize; part of Gmond interface
+
+ `params` is a dictionary of configuration options, generated by
+ Ganglia out of values specified in the module's .pyconf file. It
+ should contain an 'endpoint' key, specifying the address of the
+ streaming endpoint. Example:
+
+ param endpoint {
+ value = 'tcp://127.0.0.1:8423'
+ }
+
+ """
+ endpoint = params['endpoint']
+ counter = {name: 0 for pattern, name, description in patterns}
+
+ thread = threading.Thread(target=count_errors, args=(counter, endpoint))
+ thread.daemon = True
+ thread.start()
+
+ time.sleep(2)
+
+ return [{
+ 'name': name,
+ 'value_type': 'uint',
+ 'format': '%d',
+ 'units': 'errors',
+ 'slope': 'positive',
+ 'time_max': 15,
+ 'description': description,
+ 'groups': 'mediawiki',
+ 'call_back': counter.get,
+ } for pattern, name, description in patterns]
+
+
+def metric_cleanup():
+ """Teardown; part of Gmond interface"""
+ pass
+
+
+if __name__ == '__main__':
+ # Self-test: report metrics to stdout every 10 seconds.
+ import sys
+
+ if len(sys.argv) != 2:
+ sys.exit('Usage: %s tcp://HOST:PORT' % __file__)
+
+ params = {'endpoint': sys.argv[1]}
+ metrics = metric_init(params)
+
+ print('Streaming errors from %(endpoint)s...' % params)
+
+ while 1:
+ print('\n{:-^32}'.format(time.asctime()))
+ for metric in metrics:
+ call_back = metric['call_back']
+ name = metric['name']
+ description = metric['description']
+ print('{:.<30}{}'.format(description, call_back(name)))
+ time.sleep(10)
+
+# vim: set et ft=python ts=4 sw=4:
diff --git a/metric_module/mwerrors.pyconf.sample
b/metric_module/mwerrors.pyconf.sample
new file mode 100644
index 0000000..e3949ae
--- /dev/null
+++ b/metric_module/mwerrors.pyconf.sample
@@ -0,0 +1,52 @@
+/**
+ * MediaWiki exceptions & fatals monitoring
+ * File managed by Puppet
+ */
+
+modules {
+ module {
+ name = "mwerrors"
+ language = "python"
+ param endpoint {
+ value = "tcp://127.0.0.1:8423"
+ }
+ }
+}
+
+
+collection_group {
+
+ collect_every = 15
+ time_threshold = 30
+
+ metric {
+ name = "oom"
+ title = "Out-of-memory fatals"
+ value_threshold = 1
+ }
+ metric {
+ name = "timelimit"
+ title = "Time limit fatals"
+ value_threshold = 1
+ }
+ metric {
+ name = "fatal"
+ title = "Miscellaneous fatals"
+ value_threshold = 1
+ }
+ metric {
+ name = "exception"
+ title = "Exceptions"
+ value_threshold = 1
+ }
+ metric {
+ name = "catchable"
+ title = "Catchable fatals"
+ value_threshold = 1
+ }
+ metric {
+ name = "query"
+ title = "Query errors"
+ value_threshold = 1
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/73891
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic4990efa56b9af31f6416211b66c311fe9f4ffbb
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/tools/fluoride
Gerrit-Branch: master
Gerrit-Owner: Ori.livneh <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
