This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new c497078 METRON-2013 The bro plugin docker script's topic name
should be configurable (JonZeolla via ottobackwards) closes
apache/metron-bro-plugin-kafka#27
c497078 is described below
commit c49707874284dec11c37af138ffd99d503df9836
Author: JonZeolla <[email protected]>
AuthorDate: Thu Feb 28 15:48:32 2019 -0500
METRON-2013 The bro plugin docker script's topic name should be
configurable (JonZeolla via ottobackwards) closes
apache/metron-bro-plugin-kafka#27
---
docker/README.md | 18 ++++++++------
docker/run_end_to_end.sh | 26 ++++++++++++++------
...et_bro_kafka.sh => docker_run_consume_kafka.sh} | 28 +++++++++++++++++++---
...afka.sh => docker_run_create_topic_in_kafka.sh} | 15 ++++++++++--
...bro_kafka.sh => docker_run_get_offset_kafka.sh} | 18 +++++++-------
5 files changed, 77 insertions(+), 28 deletions(-)
diff --git a/docker/README.md b/docker/README.md
index 3bae93b..a965d8b 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -71,9 +71,9 @@ testing scripts to be added to a pull request, and
subsequently to a test suite.
├── docker_execute_process_data_file.sh
├── docker_execute_shell.sh
├── docker_run_bro_container.sh
-├── docker_run_consume_bro_kafka.sh
-├── docker_run_create_bro_topic_in_kafka.sh
-├── docker_run_get_offset_bro_topic_in_kafka.sh
+├── docker_run_consume_kafka.sh
+├── docker_run_create_topic_in_kafka.sh
+├── docker_run_get_offset_kafka.sh
├── docker_run_kafka_container.sh
├── docker_run_wait_for_kafka.sh
├── docker_run_wait_for_zookeeper.sh
@@ -146,22 +146,25 @@ testing scripts to be added to a pull request, and
subsequently to a test suite.
> You can then execute these scripts or use them together as part of testing
etc. by creating `docker execute` scripts like those here.
> The goal is to allow an individual to use and maintain their own library
of scripts to use instead of, or in concert with the scripts maintained by this
project.
-- `docker_run_consume_bro_kafka.sh`: Runs an instance of the kafka container,
with the console consumer `kafka-console-consumer.sh --topic bro --offset
$OFFSET --partition 0 --bootstrap-server kafka:9092`
+- `docker_run_consume_kafka.sh`: Runs an instance of the kafka container, with
the console consumer `kafka-console-consumer.sh --topic $KAFKA_TOPIC --offset
$OFFSET --partition 0 --bootstrap-server kafka:9092`
###### Parameters
```bash
--network-name [OPTIONAL] The Docker network name. Default:
bro-network
--offset [OPTIONAL] The kafka offset. Default: -1
+ --kafka-topic [OPTIONAL] The kafka topic to consume from.
Default: bro
```
-- `docker_run_get_offset_bro_kafka.sh`: Runs an instance of the kafka
container and gets the current offset for the bro topic
+- `docker_run_get_offset_kafka.sh`: Runs an instance of the kafka container
and gets the current offset for the specified topic
###### Parameters
```bash
--network-name [OPTIONAL] The Docker network name. Default:
bro-network
--offset [OPTIONAL] The kafka offset. Default: -1
+ --kafka-topic [OPTIONAL] The kafka topic to get the offset
from. Default: bro
```
-- `docker_run_create_bro_topic_in_kafka.sh`: Runs an instance of the kafka
container, creating the `bro` topic
+- `docker_run_create_topic_in_kafka.sh`: Runs an instance of the kafka
container, creating the specified topic
###### Parameters
```bash
--network-name [OPTIONAL] The Docker network name. Default:
bro-network
+ --kafka-topic [OPTIONAL] The kafka topic to create.
Default: bro
```
- `docker_run_kafka_container.sh`: Runs the main instance of the kafka
container in the background
###### Parameters
@@ -223,7 +226,7 @@ This script does the following:
3. Waits for zookeeper to be available
4. Runs the kafka container
5. Waits for kafka to be available
-6. Creates the bro topic
+6. Creates the specified topic
7. Downloads sample PCAP data
8. Runs the bro container in the background
@@ -294,4 +297,5 @@ Other scripts may then be used to do your testing, for
example running:
```bash
--skip-docker-build [OPTIONAL] Skip build of bro docker machine.
--data-path [OPTIONAL] The pcap data path. Default: ./data
+--kafka-topic [OPTIONAL] The kafka topic name to use.
Default: bro
```
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
index a1dfb6a..6dfd146 100755
--- a/docker/run_end_to_end.sh
+++ b/docker/run_end_to_end.sh
@@ -28,6 +28,7 @@ function help {
echo "USAGE"
echo " --skip-docker-build [OPTIONAL] Skip build of bro
docker machine."
echo " --data-path [OPTIONAL] The pcap data path.
Default: ./data"
+ echo " --kafka-topic [OPTIONAL] The kafka topic to
consume from. Default: bro"
echo " -h/--help Usage information."
echo " "
echo "COMPATABILITY"
@@ -51,10 +52,11 @@ DATA_PATH="${ROOT_DIR}"/data
DATE=$(date)
LOG_DATE=${DATE// /_}
TEST_OUTPUT_PATH="${ROOT_DIR}/test_output/"${LOG_DATE//:/_}
+KAFKA_TOPIC="bro"
+
# Handle command line options
for i in "$@"; do
case $i in
-
#
# SKIP_REBUILD_BRO
#
@@ -64,7 +66,6 @@ for i in "$@"; do
SKIP_REBUILD_BRO=true
shift # past argument
;;
-
#
# DATA_PATH
#
@@ -72,7 +73,15 @@ for i in "$@"; do
DATA_PATH="${i#*=}"
shift # past argument=value
;;
-
+ #
+ # KAFKA_TOPIC
+ #
+ # --kafka-topic
+ #
+ --kafka-topic=*)
+ KAFKA_TOPIC="${i#*=}"
+ shift # past argument=value
+ ;;
#
# -h/--help
#
@@ -88,6 +97,8 @@ EXTRA_ARGS="$*"
echo "Running build_container with "
echo "SKIP_REBUILD_BRO = $SKIP_REBUILD_BRO"
+echo "DATA_PATH = $DATA_PATH"
+echo "KAFKA_TOPIC = $KAFKA_TOPIC"
echo "==================================================="
# Create the network
@@ -120,8 +131,8 @@ rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc}
fi
-# Create the bro topic
-bash "${SCRIPT_DIR}"/docker_run_create_bro_topic_in_kafka.sh
+# Create the kafka topic
+bash "${SCRIPT_DIR}"/docker_run_create_topic_in_kafka.sh
--kafka-topic=${KAFKA_TOPIC}
rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc}
fi
@@ -187,7 +198,7 @@ do
# get the current offset in kafka
# this is where we are going to _start_
- OFFSET=$(bash "${SCRIPT_DIR}"/docker_run_get_offset_bro_kafka.sh | sed
's/^bro:0:\(.*\)$/\1/')
+ OFFSET=$(bash "${SCRIPT_DIR}"/docker_run_get_offset_kafka.sh
--kafka-topic=${KAFKA_TOPIC} | sed "s/^${KAFKA_TOPIC}:0:\(.*\)$/\1/")
echo "OFFSET------------------> ${OFFSET}"
bash "${SCRIPT_DIR}"/docker_execute_process_data_file.sh
--pcap-file-name="${BASE_FILE_NAME}"
--output-directory-name="${DOCKER_DIRECTORY_NAME}"
@@ -197,7 +208,8 @@ do
fi
KAFKA_OUTPUT_FILE="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}/kafka-output.log"
- bash "${SCRIPT_DIR}"/docker_run_consume_bro_kafka.sh --offset=$OFFSET |
"${ROOT_DIR}"/remove_timeout_message.sh | tee "${KAFKA_OUTPUT_FILE}"
+ bash "${SCRIPT_DIR}"/docker_run_consume_kafka.sh --offset=${OFFSET}
--kafka-topic=${KAFKA_TOPIC} | "${ROOT_DIR}"/remove_timeout_message.sh | tee
"${KAFKA_OUTPUT_FILE}"
+
rc=$?; if [[ ${rc} != 0 ]]; then
echo "ERROR> FAILED TO PROCESS ${DATA_PATH} DATA. CHECK LOGS"
fi
diff --git a/docker/scripts/docker_run_get_offset_bro_kafka.sh
b/docker/scripts/docker_run_consume_kafka.sh
similarity index 69%
rename from docker/scripts/docker_run_get_offset_bro_kafka.sh
rename to docker/scripts/docker_run_consume_kafka.sh
index eafd08f..71e7cf9 100755
--- a/docker/scripts/docker_run_get_offset_bro_kafka.sh
+++ b/docker/scripts/docker_run_consume_kafka.sh
@@ -24,19 +24,23 @@ set -E # errtrap
set -o pipefail
#
-# Runs a kafka container with the console consumer for the bro topic. The
consumer should quit when it has read
-# all of the messages available
+# Runs a kafka container with the console consumer for the appropriate topic.
+# The consumer should quit when it has read all of the messages available.
#
function help {
echo " "
echo "usage: ${0}"
echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
+ echo " --offset [OPTIONAL] The kafka offset to
read from. Default: -1"
+ echo " --kafka-topic [OPTIONAL] The kafka topic to
consume from. Default: bro"
echo " -h/--help Usage information."
echo " "
}
NETWORK_NAME=bro-network
+OFFSET=-1
+KAFKA_TOPIC=bro
# handle command line options
for i in "$@"; do
@@ -51,6 +55,24 @@ for i in "$@"; do
shift # past argument=value
;;
#
+ # OFFSET
+ #
+ # --offset
+ #
+ --offset=*)
+ OFFSET="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
+ # KAFKA_TOPIC
+ #
+ # --kafka-topic
+ #
+ --kafka-topic=*)
+ KAFKA_TOPIC="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
# -h/--help
#
-h | --help)
@@ -71,7 +93,7 @@ for i in "$@"; do
done
docker run --rm --network "${NETWORK_NAME}" ches/kafka \
- kafka-run-class.sh kafka.tools.GetOffsetShell --topic bro --broker-list
kafka:9092
+ kafka-console-consumer.sh --topic "${KAFKA_TOPIC}" --offset "${OFFSET}"
--partition 0 --bootstrap-server kafka:9092 --timeout-ms 1000
rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc}
fi
diff --git a/docker/scripts/docker_run_create_bro_topic_in_kafka.sh
b/docker/scripts/docker_run_create_topic_in_kafka.sh
similarity index 81%
rename from docker/scripts/docker_run_create_bro_topic_in_kafka.sh
rename to docker/scripts/docker_run_create_topic_in_kafka.sh
index f4c84ab..f140469 100755
--- a/docker/scripts/docker_run_create_bro_topic_in_kafka.sh
+++ b/docker/scripts/docker_run_create_topic_in_kafka.sh
@@ -27,11 +27,13 @@ function help {
echo " "
echo "usage: ${0}"
echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
+ echo " --kafka-topic [OPTIONAL] The kafka topic to
create. Default: bro"
echo " -h/--help Usage information."
echo " "
}
NETWORK_NAME=bro-network
+KAFKA_TOPIC=bro
# handle command line options
for i in "$@"; do
@@ -46,6 +48,15 @@ for i in "$@"; do
shift # past argument=value
;;
#
+ # KAFKA_TOPIC
+ #
+ # --kafka-topic
+ #
+ --kafka-topic=*)
+ KAFKA_TOPIC="${i#*=}"
+ shift # past argument=value
+ ;;
+ #
# -h/--help
#
-h | --help)
@@ -65,12 +76,12 @@ for i in "$@"; do
esac
done
-echo "Running docker_run_create_bro_topic_in_kafka with "
+echo "Running docker_run_create_topic_in_kafka with "
echo "NETWORK_NAME = $NETWORK_NAME"
echo "==================================================="
docker run --rm --network "${NETWORK_NAME}" ches/kafka \
- kafka-topics.sh --create --topic bro --replication-factor 1 --partitions 1
--zookeeper zookeeper:2181
+ kafka-topics.sh --create --topic "${KAFKA_TOPIC}" --replication-factor 1
--partitions 1 --zookeeper zookeeper:2181
rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc}
fi
diff --git a/docker/scripts/docker_run_consume_bro_kafka.sh
b/docker/scripts/docker_run_get_offset_kafka.sh
similarity index 79%
rename from docker/scripts/docker_run_consume_bro_kafka.sh
rename to docker/scripts/docker_run_get_offset_kafka.sh
index 5a7bbcf..7d658fc 100755
--- a/docker/scripts/docker_run_consume_bro_kafka.sh
+++ b/docker/scripts/docker_run_get_offset_kafka.sh
@@ -24,21 +24,21 @@ set -E # errtrap
set -o pipefail
#
-# Runs a kafka container with the console consumer for the bro topic. The
consumer should quit when it has read
-# all of the messages available
+# Runs a kafka container with the console consumer for the provided topic. The
+# consumer should quit when it has read all of the messages available.
#
function help {
echo " "
echo "usage: ${0}"
echo " --network-name [OPTIONAL] The Docker network
name. Default: bro-network"
- echo " --offset [OPTIONAL] The kafka offset to
read from. Default: -1"
+ echo " --kafka-topic [OPTIONAL] The kafka topic to pull
the offset from. Default: bro"
echo " -h/--help Usage information."
echo " "
}
NETWORK_NAME=bro-network
-OFFSET=-1
+KAFKA_TOPIC=bro
# handle command line options
for i in "$@"; do
@@ -53,12 +53,12 @@ for i in "$@"; do
shift # past argument=value
;;
#
- # OFFSET
+ # KAFKA_TOPIC
#
- # --offset
+ # --kafka-topic
#
- --offset=*)
- OFFSET="${i#*=}"
+ --kafka-topic=*)
+ KAFKA_TOPIC="${i#*=}"
shift # past argument=value
;;
#
@@ -82,7 +82,7 @@ for i in "$@"; do
done
docker run --rm --network "${NETWORK_NAME}" ches/kafka \
- kafka-console-consumer.sh --topic bro --offset "${OFFSET}" --partition 0
--bootstrap-server kafka:9092 --timeout-ms 1000
+ kafka-run-class.sh kafka.tools.GetOffsetShell --topic "${KAFKA_TOPIC}"
--broker-list kafka:9092
rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc}
fi
