JonZeolla commented on a change in pull request #36: METRON-2069 Add btests for
bro plugin topic_name selection
URL:
https://github.com/apache/metron-bro-plugin-kafka/pull/36#discussion_r323011857
##########
File path: docker/run_end_to_end.sh
##########
@@ -196,58 +206,58 @@ rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc}
fi
+if [[ "$NO_PCAP" = false ]]; then
+ # for each pcap in the data directory, we want to
+ # run bro then read the output from kafka
+ # and output both of them to the same directory named
+ # for the date/pcap
-# for each pcap in the data directory, we want to
-# run bro then read the output from kafka
-# and output both of them to the same directory named
-# for the date/pcap
+ for file in "${DATA_PATH}"/**/*.pcap*
+ do
+ # get the file name
+ BASE_FILE_NAME=$(basename "${file}")
+ DOCKER_DIRECTORY_NAME=${BASE_FILE_NAME//\./_}
-for file in "${DATA_PATH}"/**/*.pcap*
-do
- # get the file name
- BASE_FILE_NAME=$(basename "${file}")
- DOCKER_DIRECTORY_NAME=${BASE_FILE_NAME//\./_}
+ mkdir "${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}" || exit 1
+ echo "MADE ${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}"
- mkdir "${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}" || exit 1
- echo "MADE ${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}"
+ # get the current offset in kafka
+ # this is where we are going to _start_
+ OFFSET=$(bash "${SCRIPT_DIR}"/docker_run_get_offset_kafka.sh
--kafka-topic="${KAFKA_TOPIC}" | sed "s/^${KAFKA_TOPIC}:0:\(.*\)$/\1/")
+ echo "OFFSET------------------> ${OFFSET}"
- # get the current offset in kafka
- # this is where we are going to _start_
- OFFSET=$(bash "${SCRIPT_DIR}"/docker_run_get_offset_kafka.sh
--kafka-topic="${KAFKA_TOPIC}" | sed "s/^${KAFKA_TOPIC}:0:\(.*\)$/\1/")
- echo "OFFSET------------------> ${OFFSET}"
+ bash "${SCRIPT_DIR}"/docker_execute_process_data_file.sh
--pcap-file-name="${BASE_FILE_NAME}"
--output-directory-name="${DOCKER_DIRECTORY_NAME}"
+ rc=$?; if [[ ${rc} != 0 ]]; then
+ echo "ERROR> FAILED TO PROCESS ${file} DATA. CHECK LOGS, please run the
finish_end_to_end.sh when you are done."
+ exit ${rc}
+ fi
- bash "${SCRIPT_DIR}"/docker_execute_process_data_file.sh
--pcap-file-name="${BASE_FILE_NAME}"
--output-directory-name="${DOCKER_DIRECTORY_NAME}"
- rc=$?; if [[ ${rc} != 0 ]]; then
- echo "ERROR> FAILED TO PROCESS ${file} DATA. CHECK LOGS, please run the
finish_end_to_end.sh when you are done."
- exit ${rc}
- fi
+
KAFKA_OUTPUT_FILE="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}/kafka-output.log"
+ bash "${SCRIPT_DIR}"/docker_run_consume_kafka.sh --offset="${OFFSET}"
--kafka-topic="${KAFKA_TOPIC}" | "${ROOT_DIR}"/remove_timeout_message.sh | tee
"${KAFKA_OUTPUT_FILE}"
+
+ rc=$?; if [[ ${rc} != 0 ]]; then
+ echo "ERROR> FAILED TO PROCESS ${DATA_PATH} DATA. CHECK LOGS"
+ fi
-
KAFKA_OUTPUT_FILE="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}/kafka-output.log"
- bash "${SCRIPT_DIR}"/docker_run_consume_kafka.sh --offset="${OFFSET}"
--kafka-topic="${KAFKA_TOPIC}" | "${ROOT_DIR}"/remove_timeout_message.sh | tee
"${KAFKA_OUTPUT_FILE}"
+ "${SCRIPT_DIR}"/split_kakfa_output_by_log.sh
--log-directory="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}"
Review comment:
The file also needs to be renamed to fix this typo
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
