METRON-1646 Sensor Stubs should work when kerberized (nickwallen) closes apache/metron#1087
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/81282de2 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/81282de2 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/81282de2 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: 81282de288d94b1fa8c57917ff6a34079eca77e1 Parents: 3a6fb38 Author: nickwallen <[email protected]> Authored: Thu Jun 28 14:07:32 2018 -0400 Committer: nickallen <[email protected]> Committed: Thu Jun 28 14:07:32 2018 -0400 ---------------------------------------------------------------------- .../roles/sensor-stubs/templates/start-bro-stub | 25 ++++++++++++++----- .../sensor-stubs/templates/start-snort-stub | 25 ++++++++++++++----- .../roles/sensor-stubs/templates/start-yaf-stub | 26 ++++++++++++++------ 3 files changed, 57 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/81282de2/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub b/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub index 979de3d..24027b3 100644 --- a/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub +++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub @@ -21,14 +21,20 @@ # simulates the behavior of a sensor by sending canned telemetry data # to a Kafka topic. # -# a subset of the canned data is randomly selected and is sent in +# a subset of the canned data is randomly selected and is sent in # batches. the timestamp of the message is altered to match current -# system time. the number of messages sent in each batch, along with +# system time. the number of messages sent in each batch, along with # the time delay between batches can be configured. # # start-bro-stub <DELAY> <COUNT> # +METRON_SYSCONFIG="/etc/default/metron" +if [ -f "$METRON_SYSCONFIG" ]; then + set -a + . "$METRON_SYSCONFIG" +fi + # # how long to delay between each 'batch' in seconds. # @@ -36,20 +42,27 @@ DELAY=${1:-{{ sensor_stubs_delay }}} # # how many messages to send in each 'batch'. the messages are drawn randomly -# from the entire set of canned data. +# from the entire set of canned data. # COUNT=${2:-{{ sensor_stubs_count }}} INPUT="{{ sensor_stubs_data }}/bro.out" PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh" TOPIC="bro" +SECURITY_ENABLED=${SECURITY_ENABLED:-false} +KAFKA_SECURITY_PROTOCOL=${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} + +if [ ${SECURITY_ENABLED,,} == 'true' ]; then + echo "Security enabled" + kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME +fi while true; do - + # transform the bro timestamp and push to kafka SEARCH="\"ts\"\:[0-9]\+\." REPLACE="\"ts\"\:`date +%s`\." - shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list {{ kafka_broker_url }} --topic $TOPIC - + shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list $BROKERLIST --topic $TOPIC --security-protocol $KAFKA_SECURITY_PROTOCOL + sleep $DELAY done http://git-wip-us.apache.org/repos/asf/metron/blob/81282de2/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub b/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub index 3123782..c60c002 100644 --- a/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub +++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub @@ -21,14 +21,20 @@ # simulates the behavior of a sensor by sending canned telemetry data # to a Kafka topic. # -# a subset of the canned data is randomly selected and is sent in +# a subset of the canned data is randomly selected and is sent in # batches. the timestamp of the message is altered to match current -# system time. the number of messages sent in each batch, along with +# system time. the number of messages sent in each batch, along with # the time delay between batches can be configured. # # start-snort-stub <DELAY> <COUNT> # +METRON_SYSCONFIG="/etc/default/metron" +if [ -f "$METRON_SYSCONFIG" ]; then + set -a + . "$METRON_SYSCONFIG" +fi + # # how long to delay between each 'batch' in seconds. # @@ -36,20 +42,27 @@ DELAY=${1:-{{ sensor_stubs_delay }}} # # how many messages to send in each 'batch'. the messages are drawn randomly -# from the entire set of canned data. +# from the entire set of canned data. # COUNT=${2:-{{ sensor_stubs_count }}} INPUT="{{ sensor_stubs_data }}/snort.out" PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh" TOPIC="snort" +SECURITY_ENABLED=${SECURITY_ENABLED:-false} +KAFKA_SECURITY_PROTOCOL=${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} + +if [ ${SECURITY_ENABLED,,} == 'true' ]; then + echo "Security enabled" + kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME +fi while true; do - + # transform the timestamp and push to kafka SEARCH="[^,]\+ ," REPLACE="`date +'%m\/%d\/%y-%H:%M:%S'`.000000 ," - shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list {{ kafka_broker_url }} --topic $TOPIC - + shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list $BROKERLIST --topic $TOPIC --security-protocol $KAFKA_SECURITY_PROTOCOL + sleep $DELAY done http://git-wip-us.apache.org/repos/asf/metron/blob/81282de2/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub ---------------------------------------------------------------------- diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub b/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub index 1966d39..c218c60 100644 --- a/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub +++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub @@ -21,14 +21,20 @@ # simulates the behavior of a sensor by sending canned telemetry data # to a Kafka topic. # -# a subset of the canned data is randomly selected and is sent in +# a subset of the canned data is randomly selected and is sent in # batches. the timestamp of the message is altered to match current -# system time. the number of messages sent in each batch, along with +# system time. the number of messages sent in each batch, along with # the time delay between batches can be configured. # # start-yaf-stub <DELAY> <COUNT> # +METRON_SYSCONFIG="/etc/default/metron" +if [ -f "$METRON_SYSCONFIG" ]; then + set -a + . "$METRON_SYSCONFIG" +fi + # # how long to delay between each 'batch' in seconds. # @@ -36,21 +42,27 @@ DELAY=${1:-{{ sensor_stubs_delay }}} # # how many messages to send in each 'batch'. the messages are drawn randomly -# from the entire set of canned data. +# from the entire set of canned data. # COUNT=${2:-{{ sensor_stubs_count }}} - INPUT="{{ sensor_stubs_data }}/yaf.out" PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh" TOPIC="yaf" +SECURITY_ENABLED=${SECURITY_ENABLED:-false} +KAFKA_SECURITY_PROTOCOL=${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} + +if [ ${SECURITY_ENABLED,,} == 'true' ]; then + echo "Security enabled" + kinit -kt $METRON_SERVICE_KEYTAB $METRON_PRINCIPAL_NAME +fi while true; do - + # transform the timestamp and push to kafka SEARCH="[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\} [0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}\.[0-9]\+" REPLACE="`date +'%Y-%m-%d %H:%M:%S'`.000" - shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list {{ kafka_broker_url }} --topic $TOPIC - + shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list $BROKERLIST --topic $TOPIC --security-protocol $KAFKA_SECURITY_PROTOCOL + sleep $DELAY done
