jenkins-bot has submitted this change and it was merged. Change subject: Standardize executable names; bump version to 0.6 ......................................................................
Standardize executable names; bump version to 0.6 In anticipation of a re-write of the EventLogging Puppet module, I am changing the names of the executable scripts so that they follow a clear convention (namely, starting with/'eventlogging' prefix and using full, hyphen-separated words). This change also deletes some obsoleted scripts. 'log2json', 'json2mongo', and 'json2sql' are deprecated in favor of the generic 'eventlogging-consumer', which can write to different back-ends. 'sv-alerts' is dependent on supervisor, which we're migrating away from; it needs to be replaced w/robust icinga support. Change-Id: Iada1a398e3044c44260c7f3eee74cc546e7ad08c --- M EventLogging.php R server/bin/eventlogging-consumer M server/bin/eventlogging-devserver R server/bin/eventlogging-stream-processor R server/bin/eventlogging-udp-seqid-mon R server/bin/eventlogging-udp-zmq-forwarder R server/bin/eventlogging-zmq-mux D server/bin/json2mongo D server/bin/json2sql D server/bin/sv-alerts D server/bin/zmq2log M server/setup.py 12 files changed, 26 insertions(+), 335 deletions(-) Approvals: Ori.livneh: Looks good to me, approved jenkins-bot: Verified diff --git a/EventLogging.php b/EventLogging.php index 3400422..23f1c4d 100644 --- a/EventLogging.php +++ b/EventLogging.php @@ -9,7 +9,7 @@ * * @author Ori Livneh <[email protected]> * @license GPL v2 or later - * @version 0.5 + * @version 0.6 */ // Credits @@ -23,7 +23,7 @@ 'S Page', 'Matthew Flaschen', ), - 'version' => '0.5', + 'version' => '0.6', 'url' => 'https://www.mediawiki.org/wiki/Extension:EventLogging', 'descriptionmsg' => 'eventlogging-desc' ); diff --git a/server/bin/eventlogger b/server/bin/eventlogging-consumer similarity index 82% rename from server/bin/eventlogger rename to server/bin/eventlogging-consumer index 9001b47..086560f 100755 --- a/server/bin/eventlogger +++ b/server/bin/eventlogging-consumer @@ -1,16 +1,16 @@ #!/usr/bin/env python # -*- coding: utf8 -*- """ - eventlogger - ----------- + eventlogging-consumer + --------------------- Consumes an event stream and writes it to a data store. Input stream and target data store are specified using URIs. - usage: eventlogger [-h] input output + usage: eventlogging-consumer [-h] input-uri output-uri positional arguments: - input URI of raw input stream - output URI of output stream + input-uri URI of raw input stream + output-uri URI of output stream optional arguments: -h, --help show this help message and exit diff --git a/server/bin/eventlogging-devserver b/server/bin/eventlogging-devserver index d2dfe16..76e5c7f 100755 --- a/server/bin/eventlogging-devserver +++ b/server/bin/eventlogging-devserver @@ -3,7 +3,6 @@ """ eventlogging-devserver ---------------------- - Invoking this command-line tool will spawn a web server that can serve as a test logging endpoint. Events logged against this server will be validated verbosely and pretty-printed to the terminal. diff --git a/server/bin/log2json b/server/bin/eventlogging-stream-processor similarity index 92% rename from server/bin/log2json rename to server/bin/eventlogging-stream-processor index f14e763..108ad98 100755 --- a/server/bin/log2json +++ b/server/bin/eventlogging-stream-processor @@ -1,11 +1,11 @@ #!/usr/bin/env python # -*- coding: utf8 -*- """ - log2json - -------- + eventlogging-stream-processor + ----------------------------- Transform raw log stream to JSON event stream - usage: log2json [-h] [--sid SID] format input output + usage: eventlogging-stream-processor [-h] [--sid SID] format input output positional arguments: format Format string diff --git a/server/bin/seqmon b/server/bin/eventlogging-udp-seqid-mon similarity index 95% rename from server/bin/seqmon rename to server/bin/eventlogging-udp-seqid-mon index 0eab2b7..3437d94 100755 --- a/server/bin/seqmon +++ b/server/bin/eventlogging-udp-seqid-mon @@ -1,11 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ - seqmon - ------ + eventlogging-udp-seqid-mon + -------------------------- Monitor sequence IDs on a stream of varnishncsa udp output for gaps. - usage: seqmon [options] stream destfile + usage: eventlogging-udp-seqid-mon [options] stream destfile positional arguments: stream log file or stream URI diff --git a/server/bin/udp2zmq b/server/bin/eventlogging-udp-zmq-forwarder similarity index 90% rename from server/bin/udp2zmq rename to server/bin/eventlogging-udp-zmq-forwarder index d83dbef..cfc6e91 100755 --- a/server/bin/udp2zmq +++ b/server/bin/eventlogging-udp-zmq-forwarder @@ -1,9 +1,8 @@ #!/usr/bin/env python # -*- coding: utf8 -*- """ - udp2zmq - ------- - + eventlogging-udp-zmq-forwarder + ------------------------------ UDP -> ZeroMQ socket forwarding. Reads line-oriented input from UDP socket and writes it to a ZeroMQ TCP PUB socket bound to the same port number. @@ -11,13 +10,13 @@ bytes from the UDP socket into the ZMQ socket. We use socket.makefile() to facilitate reading and writing whole lines. - usage: udp2zmq [-h] port + usage: eventlogging-udp-zmq-forwarder [-h] port positional arguments: port Port to forward optional arguments: - -h, --help show this help message and exit usage: udp2zmq.py [-h] port + -h, --help show this help message and exit :copyright: (c) 2012 by Ori Livneh <[email protected]> :license: GNU General Public Licence 2.0 or later diff --git a/server/bin/zmux b/server/bin/eventlogging-zmq-mux similarity index 92% rename from server/bin/zmux rename to server/bin/eventlogging-zmq-mux index 242777a..327cacb 100755 --- a/server/bin/zmux +++ b/server/bin/eventlogging-zmq-mux @@ -1,14 +1,14 @@ #!/usr/bin/env python # -*- coding: utf8 -*- """ - zmux - ---- + eventlogging-zmq-mux + -------------------- Multiplexes ZMQ_PUB sockets. Reads messages from one or more ZMQ_SUBs and publishes a muxed stream containing all messages from all inputs. - usage: zmux [-h] [--sid SID] output input [input ...] + usage: eventlogging-zmq-mux [-h] [--sid SID] output input [input ...] Multiplex ZMQ_PUB sockets diff --git a/server/bin/json2mongo b/server/bin/json2mongo deleted file mode 100755 index c4a69b7..0000000 --- a/server/bin/json2mongo +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - json2mongo - ---------- - - usage: json2mongo [-h] [--sid SID] input db - - MongoDB event consumer - - positional arguments: - input URI of JSON event stream to consume - db MongoDB connection string URI - - optional arguments: - -h, --help show this help message and exit - --sid SID set input socket identity - - :copyright: (c) 2012 by Ori Livneh <[email protected]> - :license: GNU General Public Licence 2.0 or later - -""" -import argparse -import logging -import sys - -from datetime import datetime - -try: - import pymongo -except ImportError: - sys.stderr.write( - 'Error: json2mongo requires pymongo. ' - 'See <http://api.mongodb.org/python/current/>.\n' - ) - sys.exit(1) - -import eventlogging - - -logging.basicConfig(stream=sys.stderr, level=logging.INFO) - -parser = argparse.ArgumentParser(description='MongoDB event consumer') -parser.add_argument('input', help='URI of JSON event stream to consume') -parser.add_argument('db', help='MongoDB URI', type=pymongo.Connection) -parser.add_argument('--sid', help='set input socket identity') -args = parser.parse_args() - -events = args.db.events -event_stream = eventlogging.zmq_subscribe(args.input, sid=args.sid, json=True) - - -while 1: # Keep ``try/except`` outside the inner loop. - try: - for count, event in enumerate(event_stream): - - if count % 100 == 0: - logging.info('Logged %d events', count) - - # Convert the integer timestamp into a Python `datetime` - # object, which pymongo serializes into a BSON `date` type. - event['timestamp'] = datetime.fromtimestamp(event['timestamp']) - - # Specify the event's UUID as the ObjectID to use for the - # event. See `<http://docs.mongodb.org/manual/core/object-id/>`. - event['_id'] = event['uuid'] - - # Unlike MySQL tables, which are specific to a schema - # revision, MongoDB collections encompass all revisions of a - # schema. This means users of the data should inspect the - # `revId` property to know against which revision of the - # schema the event was declared and validated. - events[event['schema']].insert(event) - except Exception: - logging.exception('Failed to insert event: %s', event) diff --git a/server/bin/json2sql b/server/bin/json2sql deleted file mode 100755 index 1ca66ce..0000000 --- a/server/bin/json2sql +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - json2sql - -------- - Consume JSON event stream into MySQL - - usage: json2sql [-h] [--sid SID] input db - - positional arguments: - input URI of JSON event stream to consume - db URI of DB to write to - - optional arguments: - -h, --help show this help message and exit - --sid SID set input socket identity - --debug output all generated SQL to stderr - - :copyright: (c) 2012 by Ori Livneh <[email protected]> - :license: GNU General Public Licence 2.0 or later - -""" -from __future__ import division - -import argparse -import logging -import sys - -import eventlogging -import sqlalchemy - - -parser = argparse.ArgumentParser(description='JSON event stream -> MySQL') -parser.add_argument('input', help='URI of JSON event stream to consume') -parser.add_argument('db', help='URI of DB to write to') -parser.add_argument('--sid', help='set input socket identity') -parser.add_argument('--debug', action='store_const', dest='loglevel', - const=logging.INFO, default=logging.WARNING, - help='output all generated SQL to stderr') -args = parser.parse_args() - -logging.basicConfig(stream=sys.stderr, level=args.loglevel) -logging.getLogger('sqlalchemy.engine').setLevel(args.loglevel) - -meta = sqlalchemy.MetaData(args.db) - -sub = eventlogging.zmq_subscribe(args.input, sid=args.sid, json=True) - -while 1: - # Optimization: keep ``try/except`` block outside the inner loop. - try: - for ev in sub: - logging.info(ev) - eventlogging.store_sql_event(meta, ev) - except Exception: - logging.exception('Unable to insert event: %s', ev) diff --git a/server/bin/sv-alerts b/server/bin/sv-alerts deleted file mode 100755 index bdb5e3b..0000000 --- a/server/bin/sv-alerts +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python -u -# -*- coding: utf-8 -*- -""" - sv-alerts - --------- - - E-mail supervisord alerts. - - usage: sv-alerts [-h] from to - - positional arguments: - sender sender's e-mail address - recipient recipient's e-mail address - - optional arguments: - -h, --help show this help message and exit - - :copyright: (c) 2012 by Ori Livneh <[email protected]> - :license: GNU General Public Licence 2.0 or later - -""" -from __future__ import print_function, unicode_literals - -import argparse -import ast -import datetime -import json -import logging -import subprocess -import sys - -from email.mime.text import MIMEText - - -parser = argparse.ArgumentParser(description='sv-alerts') -parser.add_argument('sender', help="sender's e-mail address") -parser.add_argument('recipient', help="recipient's e-mail address") - - -def send_email(sender, recipient, subject, text): - """Send an e-mail by shelling out to 'sendmail'.""" - message = MIMEText(text) - message['From'] = sender - message['To'] = recipient - message['Subject'] = subject - p = subprocess.Popen(('/usr/sbin/sendmail', '-t'), stdin=subprocess.PIPE) - p.communicate(message.as_string().encode('utf8')) - logging.info('sent: %s', message.as_string()) - - -def parse_tokens(raw): - """Parse a raw Supervisor event.""" - map = {} - raw = raw.strip() - for kv in raw.split(): - try: - k, v = kv.split(':') - except ValueError: - continue - try: - map[k] = ast.literal_eval(v) - except ValueError: - map[k] = v - return map - - -def iter_events(): - """Reads and yields Supervisor events.""" - while 1: - print('READY') - sys.stdout.flush() - event = parse_tokens(sys.stdin.readline()) - length = event.get('len', 0) - if length: - raw = sys.stdin.read(length) - event.update(parse_tokens(raw)) - print('RESULT 2', 'OK', sep='\n', end='') - sys.stdout.flush() - yield event - - -def send_alerts(sender, recipient): - """Listen to Supervisor events and emit e-mail alerts.""" - for event in iter_events(): - eventname = event['eventname'].replace('PROCESS_STATE_', '') - processname = event.get('processname', 'supervisord') - subject = '%s: %s' % (eventname, processname) - text = 'This is an automatic alert, generated at %s.\n\n%s' % ( - datetime.datetime.utcnow(), json.dumps(event, indent=4)) - send_email(sender, recipient, subject, text) - - -if __name__ == '__main__': - logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) - args = parser.parse_args() - send_alerts(sender=args.sender, recipient=args.recipient) diff --git a/server/bin/zmq2log b/server/bin/zmq2log deleted file mode 100755 index 5244d47..0000000 --- a/server/bin/zmq2log +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - zmq2log.py - ---------- - Log a ZeroMQ PUB stream. - - usage: zmq2log.py [-h] [--topic TOPIC] [--sid SID] publisher destfile - - Log a ZeroMQ PUB stream. - - positional arguments: - publisher publisher URI - destfile write log to this file - - optional arguments: - -h, --help show help message and exit - --topic TOPIC subscribe to topic (default: "") - --sid SID set socket identity (default: host name) - -""" -from __future__ import unicode_literals - -import argparse -import logging -import logging.handlers - -import zmq - - -# -# Parse command-line args -# - -parser = argparse.ArgumentParser(description='Log a ZeroMQ PUB stream.') -parser.add_argument('publisher', help='publisher URI') -parser.add_argument('destfile', help='write log to this file') -parser.add_argument('--topic', default='', - help='subscribe to topic (default: "")') -parser.add_argument('--sid', help='set socket identity') - -args = parser.parse_args() - - -# -# Configure logging -# - -# Configure logging to file: -logfile_handler = logging.handlers.WatchedFileHandler( - filename=args.destfile, encoding='utf8') -logfile_handler.setLevel(logging.INFO) - -# Configure logging to stderr: -console_handler = logging.StreamHandler() -console_handler.setFormatter(logging.Formatter('%(asctime)s\t%(message)s')) -console_handler.setLevel(logging.DEBUG) # Don't pollute log files with status - -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) -log.addHandler(logfile_handler) -log.addHandler(console_handler) -log.debug('Started. Logging to %s.' % args.destfile) - - -# -# Configure ZeroMQ Subscriber -# - -context = zmq.Context.instance() -socket = context.socket(zmq.SUB) -if args.sid is not None: - socket.setsockopt(zmq.IDENTITY, args.sid.encode('utf8')) -socket.connect(args.publisher) -socket.setsockopt(zmq.SUBSCRIBE, args.topic.encode('utf8')) -log.debug('Connected to %s/%s' % (args.publisher, args.topic)) - - -while 1: - log.info(socket.recv_unicode().rstrip()) diff --git a/server/setup.py b/server/setup.py index 60acccd..ea02dd2 100644 --- a/server/setup.py +++ b/server/setup.py @@ -19,7 +19,7 @@ setup( name='eventlogging', - version='0.5', + version='0.6', license='GPL', author='Ori Livneh', author_email='[email protected]', @@ -43,12 +43,12 @@ 'eventlogging', ), scripts=( - 'bin/eventlogger', + 'bin/eventlogging-consumer', 'bin/eventlogging-devserver', - 'bin/log2json', - 'bin/seqmon', - 'bin/udp2zmq', - 'bin/zmux', + 'bin/eventlogging-stream-processor', + 'bin/eventlogging-udp-seqid-mon', + 'bin/eventlogging-udp-zmq-forwarder', + 'bin/eventlogging-zmq-mux', ), zip_safe=False, test_suite='tests', -- To view, visit https://gerrit.wikimedia.org/r/70920 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Iada1a398e3044c44260c7f3eee74cc546e7ad08c Gerrit-PatchSet: 2 Gerrit-Project: mediawiki/extensions/EventLogging Gerrit-Branch: master Gerrit-Owner: Ori.livneh <[email protected]> Gerrit-Reviewer: Ori.livneh <[email protected]> Gerrit-Reviewer: jenkins-bot _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
