This is an automated email from the ASF dual-hosted git repository.
wiener pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new 3e60e56 [STREAMPIPES-174] Cannot start multiple instances of processor
3e60e56 is described below
commit 3e60e56652065a7b1e7081bd8fd0342bc1268c46
Author: Patrick Wiener <[email protected]>
AuthorDate: Mon Jul 20 07:58:09 2020 +0200
[STREAMPIPES-174] Cannot start multiple instances of processor
---
streampipes-wrapper-python/streampipes/api.py | 4 +--
.../streampipes/configuration.py | 2 +-
streampipes-wrapper-python/streampipes/core.py | 41 +++++++++-------------
streampipes-wrapper-python/streampipes/manager.py | 20 +++++++----
4 files changed, 33 insertions(+), 34 deletions(-)
diff --git a/streampipes-wrapper-python/streampipes/api.py
b/streampipes-wrapper-python/streampipes/api.py
index 39dbe6e..f6baf08 100644
--- a/streampipes-wrapper-python/streampipes/api.py
+++ b/streampipes-wrapper-python/streampipes/api.py
@@ -58,7 +58,7 @@ class API(object):
:param json: contains invocation graph.
:return: None.
"""
- return ProcessorDispatcher.start(app_id=kwargs.get('processor_id'),
**kwargs)
+ return ProcessorDispatcher.start(**kwargs)
@staticmethod
def detach(**kwargs):
@@ -67,4 +67,4 @@ class API(object):
:param json: contains elementId to be terminated
:return: None.
"""
- return
ProcessorDispatcher.stop(invocation_id=kwargs.get('invocation_id'))
\ No newline at end of file
+ return ProcessorDispatcher.stop(**kwargs)
\ No newline at end of file
diff --git a/streampipes-wrapper-python/streampipes/configuration.py
b/streampipes-wrapper-python/streampipes/configuration.py
index 1a6ae11..719b6e1 100644
--- a/streampipes-wrapper-python/streampipes/configuration.py
+++ b/streampipes-wrapper-python/streampipes/configuration.py
@@ -19,7 +19,7 @@ import os
flask_host = os.getenv('FLASK_HOST', '0.0.0.0')
port = os.getenv('SP_PORT', 5000)
-kafka_thread = 'kafka_thread'
+kafka_consumer_thread = 'kafka_consumer_thread'
banner="""\
_______ __ ______ __
diff --git a/streampipes-wrapper-python/streampipes/core.py
b/streampipes-wrapper-python/streampipes/core.py
index 346e04f..c31cd9e 100644
--- a/streampipes-wrapper-python/streampipes/core.py
+++ b/streampipes-wrapper-python/streampipes/core.py
@@ -17,13 +17,13 @@
"""contains relevant base classes"""
import json
import logging
-import queue
-import uuid
from abc import ABC, abstractmethod
+from confluent_kafka.admin import AdminClient
+
from streampipes.api import API
-from streampipes.configuration import kafka_thread, banner
-from confluent_kafka import Producer, Consumer
+from streampipes.configuration import banner, kafka_consumer_thread
from streampipes.helper import threaded
+from confluent_kafka import Producer, Consumer
class StandaloneSubmitter(ABC):
@@ -46,7 +46,6 @@ class EventProcessor(ABC):
'session.timeout.ms': 30000,
'fetch.max.bytes': 5000012,
'auto.offset.reset': 'latest',
- 'group.id': 'streampipes_python_' + str(uuid.uuid4()),
}
_DEFAULT_KAFKA_PRODUCER_CONFIG = {
@@ -59,14 +58,13 @@ class EventProcessor(ABC):
def __init__(self, **kwargs):
"""initialize EventProcessor with Kafka Prodcuer and Consumer"""
self.logger = logging.getLogger(__name__)
+
self._input_topics = kwargs.get('input_topics')
self._output_topics = kwargs.get('output_topics')
self._invocation_id = kwargs.get('invocation_id')
self._bootstrap_servers = kwargs.get('bootstrap_servers')
self.static_properties = kwargs.get('static_properties')
- # exchange events via queue - maybe register callback would be better
- self.queue = queue.Queue(maxsize=0)
self._running = False
self._threads = {}
@@ -74,21 +72,23 @@ class EventProcessor(ABC):
self._DEFAULT_KAFKA_CONSUMER_CONFIG['bootstrap.servers'] =
self._bootstrap_servers
self._DEFAULT_KAFKA_PRODUCER_CONFIG['bootstrap.servers'] =
self._bootstrap_servers
- self._producer = Producer(**self._DEFAULT_KAFKA_PRODUCER_CONFIG)
- self._consumer = Consumer(**self._DEFAULT_KAFKA_CONSUMER_CONFIG)
+ self._DEFAULT_KAFKA_CONSUMER_CONFIG['group.id'] =
'streampipes_python_' + self._invocation_id
+
+ self._producer = Producer(self._DEFAULT_KAFKA_PRODUCER_CONFIG)
+ self._consumer = Consumer(self._DEFAULT_KAFKA_CONSUMER_CONFIG)
#self._create_topic(topic=self._output_topics,
conf=self._DEFAULT_KAFKA_PRODUCER_CONFIG)
self.on_invocation()
def init(self):
- self.logger.info('start processor {}'.format(self._invocation_id))
- self._threads[kafka_thread] = self.kafka_thread_main()
+ self.logger.info('start processor {}'.format(self.invocation_id))
+ self._threads[kafka_consumer_thread] =
self._consume(self._input_topics)
def active_threads(self):
return self._threads
@property
- def invoke_id(self):
+ def invocation_id(self):
return self._invocation_id
def __del__(self):
@@ -115,18 +115,7 @@ class EventProcessor(ABC):
self._produce(result)
@threaded
- def kafka_thread_main(self):
- """ start threaded kafka consumer """
- try:
- self._consumer_run(self._input_topics)
- except KeyboardInterrupt:
- self.logger.info("consumer: aborted by user")
- self._running = False
- except Exception as ex:
- self.logger.fatal("consumer: fatal exception: {}".format(ex))
- self._running = False
-
- def _consumer_run(self, topics):
+ def _consume(self, topics):
""" retrieve events from kafka """
self._consumer.subscribe(topics=[topics])
self._running = True
@@ -143,6 +132,7 @@ class EventProcessor(ABC):
continue
else:
try:
+ # json -> dict
event = json.loads(msg.value().decode('utf-8'))
if isinstance(event, int):
self.logger.info("Integer not allowed
{}".format(event))
@@ -157,6 +147,7 @@ class EventProcessor(ABC):
""" send events to kafka """
event = json.dumps(result).encode('utf-8')
try:
+ # dict -> json
self._producer.produce(self._output_topics, value=event)
except BufferError:
self._producer.poll(1)
@@ -175,7 +166,7 @@ class EventProcessor(ABC):
# raise
def stop(self):
- self.logger.info('stop processor {}'.format(self._invocation_id))
+ self.logger.info('stop processor {}'.format(self.invocation_id))
self._running = False
self._consumer.close()
self._producer.flush()
diff --git a/streampipes-wrapper-python/streampipes/manager.py
b/streampipes-wrapper-python/streampipes/manager.py
index d7a2d50..64a2a4e 100644
--- a/streampipes-wrapper-python/streampipes/manager.py
+++ b/streampipes-wrapper-python/streampipes/manager.py
@@ -17,7 +17,8 @@
"""Manages processor life cycle"""
import logging
from abc import ABC
-from streampipes.configuration import kafka_thread
+
+from streampipes.configuration import kafka_consumer_thread
class Declarer(ABC):
@@ -40,33 +41,40 @@ class Declarer(ABC):
class ProcessorDispatcher(ABC):
_running_instances = {}
+
logger = logging.getLogger(__name__)
@classmethod
- def start(cls, processor_id=None, **kwargs):
+ def start(cls, **kwargs):
+ processor_id = kwargs.get('processor_id')
try:
processor = Declarer.get_processor(processor_id)(**kwargs)
processor.init()
- cls._running_instances[processor.invoke_id] = processor
+ cls._running_instances[processor.invocation_id] = processor
+
return {'status': 'success'}
except KeyError:
err = "KeyError. processor_id not found"
- cls.logger.info('{}: {}'.format(err,processor_id))
+ cls.logger.info('{}: {}'.format(err, processor_id))
return {'status': err}
@classmethod
- def stop(cls, invocation_id=None):
+ def stop(cls, **kwargs):
+ invocation_id = kwargs.get('invocation_id')
try:
processor = cls._running_instances[invocation_id]
active_threads = processor.active_threads()
processor.stop()
- active_threads[kafka_thread].join()
+ active_threads[kafka_consumer_thread].join()
+
del processor
cls._running_instances.pop(invocation_id)
+
return {'status': 'success'}
except KeyError:
err = "KeyError. invocation_id not found"
cls.logger.info('{}: {}'.format(err,invocation_id))
+
return {'status': err}
\ No newline at end of file