Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/321397
Change subject: Remove ensure-kafka-topcis-exist script and dependency on
pykafka
......................................................................
Remove ensure-kafka-topcis-exist script and dependency on pykafka
Change-Id: I3746519358731592cae0ea42ab881d98d3aeda33
---
D bin/ensure-kafka-topics-exist
M eventlogging/utils.py
M requirements.txt
M setup.py
4 files changed, 0 insertions(+), 93 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/eventlogging
refs/changes/97/321397/1
diff --git a/bin/ensure-kafka-topics-exist b/bin/ensure-kafka-topics-exist
deleted file mode 100755
index 63b74cc..0000000
--- a/bin/ensure-kafka-topics-exist
+++ /dev/null
@@ -1,90 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import argparse
-import logging
-import sys
-import yaml
-
-from pykafka import KafkaClient
-
-ap = argparse.ArgumentParser(
- description='Ensures all topics in a topic config file exist in Kafka.',
- fromfile_prefix_chars='@'
-)
-
-ap.add_argument(
- '--topic-config',
- default='./config/schemas/config/eventbus-topics.yaml',
- help='Topic -> schema config file',
-)
-
-ap.add_argument(
- '--brokers',
- default='localhost:9092',
- help='Comma-separated list of kafka broker hosts',
-)
-
-ap.add_argument(
- '--prefixes',
- default='',
- help='Comma-separated list of topic prefixes to use',
-)
-
-ap.add_argument(
- '--no-prefix',
- action='store_true',
- help='Don\'t use topic name prefixes',
-)
-
-
-def ensure_topics_exist(kafka_client, topics, prefixes):
- if prefixes:
- topics = [dc_prefix + '.' + topic
- for topic in topics for dc_prefix in prefixes]
- for topic in topics:
- # Accessing the topic here will create it if
- # it doesn't already exist.
- t = kafka_client.topics[topic]
- logging.info(
- 'Kafka topic %s exists with %d partition(s).',
- topic,
- len(t.partitions.keys())
- )
-
-
-if __name__ == "__main__":
- logging.basicConfig(
- stream=sys.stderr,
- level=logging.INFO,
- format='%(asctime)s %(message)s'
- )
- # PyKafka can be a little noisy.
- logging.getLogger("pykafka").setLevel(logging.WARN)
-
- args = ap.parse_args()
-
- logging.info(
- 'Ensuring all topics configured in %s exist in Kafka'
- 'cluster including brokers %s',
- args.topic_config, args.brokers
- )
-
- topic_config = None
- # Load the topic_config from the config file.
- with open(args.topic_config) as f:
- topic_config = yaml.load(f)
-
- # check the topic name prefixes
- if not args.prefixes and not args.no_prefix:
- logging.error(
- 'You must specify the topic name prefixes or use --no-prefix!'
- )
- sys.exit(1)
-
- kafka_client = KafkaClient(hosts=args.brokers)
- if args.no_prefix:
- args.prefixes = []
- else:
- args.prefixes = args.prefixes.split(',')
- ensure_topics_exist(kafka_client, topic_config.keys(), args.prefixes)
diff --git a/eventlogging/utils.py b/eventlogging/utils.py
index 825f54e..a0cfa62 100644
--- a/eventlogging/utils.py
+++ b/eventlogging/utils.py
@@ -290,5 +290,4 @@
# Set module logging level to INFO, DEBUG is too noisy.
logging.getLogger("kafka").setLevel(logging.INFO)
- logging.getLogger("pykafka").setLevel(logging.INFO)
logging.getLogger("kazoo").setLevel(logging.INFO)
diff --git a/requirements.txt b/requirements.txt
index 0597076..44a567b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,6 @@
kafka-python>=1.3.0
mysqlclient>=1.3.7
pygments>=1.5
-pykafka>=2.1.0
psutil>=2.1.1
PyYAML>=3.10
pyzmq>=2.1
diff --git a/setup.py b/setup.py
index a65d67e..2ed421f 100644
--- a/setup.py
+++ b/setup.py
@@ -58,7 +58,6 @@
'bin/eventlogging-processor',
'bin/eventlogging-reporter',
'bin/eventlogging-service',
- 'bin/ensure-kafka-topics-exist',
),
zip_safe=False,
test_suite='eventlogging.tests',
--
To view, visit https://gerrit.wikimedia.org/r/321397
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I3746519358731592cae0ea42ab881d98d3aeda33
Gerrit-PatchSet: 1
Gerrit-Project: eventlogging
Gerrit-Branch: master
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
