This is an automated email from the ASF dual-hosted git repository.
jonzeolla pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new 587e9da METRON-2270 Update to reflect bro project rename (JonZeolla)
closes apache/metron-bro-plugin-kafka#44
587e9da is described below
commit 587e9dac9beeee915f7d0c1116a4432447b5d92e
Author: JonZeolla <zeo...@gmail.com>
AuthorDate: Tue May 5 20:07:34 2020 -0400
METRON-2270 Update to reflect bro project rename (JonZeolla) closes
apache/metron-bro-plugin-kafka#44
---
CMakeLists.txt | 28 ++---
Makefile | 2 +-
README.md | 132 +++++++++++----------
bro-pkg.meta | 15 ---
configure | 122 +++++++++++++++----
configure.plugin | 9 +-
.../release-utils/metron-bro-kafka-rc-check | 34 +++---
docker/README.md | 75 ++++++------
docker/containers/bro/requirements-to-freeze.txt | 1 -
docker/containers/{bro => zeek}/.screenrc | 0
docker/containers/{bro => zeek}/Dockerfile | 44 ++++---
docker/containers/{bro => zeek}/Makefile | 0
docker/containers/zeek/requirements-to-freeze.txt | 1 +
docker/containers/{bro => zeek}/requirements.txt | 4 +-
docker/docker-compose.yml | 10 +-
.../{build_bro_plugin.sh => build_plugin.sh} | 41 ++++---
...configure_bro_plugin.sh => configure_plugin.sh} | 20 ++--
docker/in_docker_scripts/process_data_file.sh | 2 +-
docker/run_end_to_end.sh | 38 +++---
docker/scripts/analyze_results.sh | 6 +-
...ro_plugin.sh => docker_execute_build_plugin.sh} | 12 +-
...lugin.sh => docker_execute_configure_plugin.sh} | 16 +--
....sh => docker_execute_configure_zeek_plugin.sh} | 16 +--
.../docker_execute_create_topic_in_kafka.sh | 4 +-
docker/scripts/docker_execute_process_data_file.sh | 6 +-
docker/scripts/docker_execute_shell.sh | 4 +-
docker/scripts/docker_run_consume_kafka.sh | 4 +-
docker/scripts/docker_run_get_offset_kafka.sh | 4 +-
docker/scripts/download_sample_pcaps.sh | 4 +-
docker/scripts/split_kafka_output_by_log.sh | 10 +-
.../Apache/Kafka/{__load__.bro => __load__.zeek} | 2 +-
.../{logs-to-kafka.bro => logs-to-kafka.zeek} | 4 +-
scripts/{__load__.bro => __load__.zeek} | 13 +-
scripts/{init.bro => init.zeek} | 6 +-
src/KafkaWriter.cc | 14 +--
src/KafkaWriter.h | 5 +-
src/Plugin.h | 4 +-
src/TaggedJSON.h | 5 +-
tests/.gitignore | 2 +
tests/Baseline/kafka.resolved-topic-default/output | 2 +-
.../Scripts/diff-remove-timestamps | 22 ++--
tests/Scripts/get-bro-env | 36 ------
tests/Scripts/get-zeek-env | 51 ++++++++
tests/btest.cfg | 14 +--
...-l2e-no-overlap.bro => l2s-l2e-no-overlap.zeek} | 2 +-
.../{l2s-set-l2e-set.bro => l2s-set-l2e-set.zeek} | 2 +-
...2s-set-l2e-unset.bro => l2s-set-l2e-unset.zeek} | 2 +-
...2s-unset-l2e-set.bro => l2s-unset-l2e-set.zeek} | 2 +-
...nset-l2e-unset.bro => l2s-unset-l2e-unset.zeek} | 2 +-
...topic-config.bro => resolved-topic-config.zeek} | 2 +-
...pic-default.bro => resolved-topic-default.zeek} | 2 +-
...bro => resolved-topic-override-and-config.zeek} | 4 +-
...-only.bro => resolved-topic-override-only.zeek} | 4 +-
...e-set.bro => send-all-active-logs-l2e-set.zeek} | 2 +-
...set.bro => send-all-active-logs-l2e-unset.zeek} | 2 +-
...o => send-all-active-logs-l2s-set-l2e-set.zeek} | 2 +-
...=> send-all-active-logs-l2s-set-l2e-unset.zeek} | 2 +-
tests/kafka/{show-plugin.bro => show-plugin.zeek} | 2 +-
tests/random.seed | 21 ++++
zkg.meta | 15 +++
60 files changed, 528 insertions(+), 384 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 18b8a14..5d8613a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -15,25 +15,25 @@
# limitations under the License.
#
-cmake_minimum_required(VERSION 2.8)
-project(Plugin)
-include(BroPlugin)
+cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
+project(ZeekPlugin_Kafka)
+include(ZeekPlugin)
find_package(LibRDKafka)
find_package(OpenSSL)
if (LIBRDKAFKA_FOUND AND OPENSSL_FOUND)
include_directories(BEFORE ${LibRDKafka_INCLUDE_DIR} ${OpenSSL_INCLUDE_DIR})
- bro_plugin_begin(APACHE KAFKA)
- bro_plugin_cc(src/KafkaWriter.cc)
- bro_plugin_cc(src/Plugin.cc)
- bro_plugin_cc(src/TaggedJSON.cc)
- bro_plugin_bif(src/kafka.bif)
- bro_plugin_bif(src/events.bif)
- bro_plugin_dist_files(README CHANGES COPYING VERSION)
- bro_plugin_link_library(${LibRDKafka_LIBRARIES})
- bro_plugin_link_library(${LibRDKafka_C_LIBRARIES})
- bro_plugin_link_library(${OpenSSL_LIBRARIES})
- bro_plugin_end()
+ zeek_plugin_begin(APACHE KAFKA)
+ zeek_plugin_cc(src/KafkaWriter.cc)
+ zeek_plugin_cc(src/Plugin.cc)
+ zeek_plugin_cc(src/TaggedJSON.cc)
+ zeek_plugin_bif(src/kafka.bif)
+ zeek_plugin_bif(src/events.bif)
+ zeek_plugin_dist_files(README CHANGES COPYING VERSION)
+ zeek_plugin_link_library(${LibRDKafka_LIBRARIES})
+ zeek_plugin_link_library(${LibRDKafka_C_LIBRARIES})
+ zeek_plugin_link_library(${OpenSSL_LIBRARIES})
+ zeek_plugin_end()
elseif (NOT LIBRDKAFKA_FOUND)
message(FATAL_ERROR "LibRDKafka not found.")
diff --git a/Makefile b/Makefile
index 50fa3ca..982db4c 100644
--- a/Makefile
+++ b/Makefile
@@ -25,7 +25,7 @@ all: build-it
build-it:
@test -e $(cmake_build_dir)/config.status || ./configure
-@test -e $(cmake_build_dir)/CMakeCache.txt && \
- test $(cmake_build_dir)/CMakeCache.txt -ot `cat
$(cmake_build_dir)/CMakeCache.txt | grep BRO_DIST | cut -d '=' -f
2`/build/CMakeCache.txt && \
+ test $(cmake_build_dir)/CMakeCache.txt -ot `cat
$(cmake_build_dir)/CMakeCache.txt | grep ZEEK_DIST | cut -d '=' -f
2`/build/CMakeCache.txt && \
echo Updating stale CMake cache && \
touch $(cmake_build_dir)/CMakeCache.txt
diff --git a/README.md b/README.md
index 72436e9..7ffbac0 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,8 @@
-# Logging Bro Output to Kafka
+# Logging Zeek Output to Kafka
-A Bro log writer that sends logging output to Kafka. This provides a
convenient means for tools in the Hadoop ecosystem, such as Storm, Spark, and
others, to process the data generated by Bro.
+A Zeek log writer that sends logging output to Kafka. This provides a
convenient means for tools in the Hadoop ecosystem, such as Storm, Spark, and
others, to process the data generated by Zeek.
-This software is a part of the [Apache Metron](http://metron.apache.org/)
project which integrates a variety of open source, big data technologies to
offer a platform to detect and respond to cyber threats at-scale.
+This software is a part of the [Apache Metron](https://metron.apache.org/)
project which integrates a variety of open source, big data technologies to
offer a platform to detect and respond to cyber threats at-scale.
* [Installation](#installation)
* [Activation](#activation)
@@ -12,68 +12,68 @@ This software is a part of the [Apache
Metron](http://metron.apache.org/) projec
## Installation
-### `bro-pkg` Installation
+### `zkg` Installation
-`bro-pkg` is the preferred mechanism for installing this plugin, as it will
dynamically retrieve, build, test, and load the plugin. Note, that you will
still need to [activate](#activation) and configure the plugin after your
installation.
+`zkg` is the preferred mechanism for installing this plugin, as it will
dynamically retrieve, build, test, and load the plugin. Note, that you will
still need to [activate](#activation) and configure the plugin after your
installation.
-1. Install [librdkafka](https://github.com/edenhill/librdkafka), a native
client library for Kafka. This plugin has been tested against the latest
release of librdkafka, which at the time of this writing is v0.11.5.
+1. Install [librdkafka](https://github.com/edenhill/librdkafka), a native
client library for Kafka. This plugin has been tested against librdkafka
v1.4.2-RC3.
In order to use this plugin within a kerberized Kafka environment, you
will also need `libsasl2` installed and will need to pass `--enable-sasl` to
the `configure` script.
```
- $ curl -L https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz |
tar xvz
- $ cd librdkafka-0.11.5/
+ $ curl -L https://github.com/edenhill/librdkafka/archive/v1.4.2-RC3.tar.gz
| tar xvz
+ $ cd librdkafka-1.4.2-RC3/
$ ./configure --enable-sasl
$ make
$ sudo make install
```
-1. Configure `bro-pkg` by following the quickstart guide
[here](https://bro-package-manager.readthedocs.io/en/stable/quickstart.html).
+1. Configure `zkg` by following the quickstart guide
[here](https://docs.zeek.org/projects/package-manager/en/stable/quickstart.html).
-1. Install the plugin using `bro-pkg install`.
+1. Install the plugin using `zkg install`.
```
- $ bro-pkg install apache/metron-bro-plugin-kafka --version master
+ $ zkg install apache/metron-bro-plugin-kafka --version master
The following packages will be INSTALLED:
- bro/apache/metron-bro-plugin-kafka (master)
+ zeek/apache/metron-bro-plugin-kafka (master)
Verify the following REQUIRED external dependencies:
(Ensure their installation on all relevant systems before proceeding):
- from bro/apache/metron-bro-plugin-kafka (master):
- librdkafka ~0.11.5
+ from zeek/apache/metron-bro-plugin-kafka (master):
+ librdkafka ~1.4.2-RC3
Proceed? [Y/n]
- bro/apache/metron-bro-plugin-kafka asks for LIBRDKAFKA_ROOT (Path to
librdkafka installation tree) ? [/usr/local/lib]
- Saved answers to config file: /home/jonzeolla/.bro-pkg/config
- Running unit tests for "bro/apache/metron-bro-plugin-kafka"
+ zeek/apache/metron-bro-plugin-kafka asks for LIBRDKAFKA_ROOT (Path to
librdkafka installation tree) ? [/usr/local/lib]
+ Saved answers to config file: /home/jonzeolla/.zkg/config
+ Running unit tests for "zeek/apache/metron-bro-plugin-kafka"
all 10 tests successful
- Installing "bro/apache/metron-bro-plugin-kafka"........
- Installed "bro/apache/metron-bro-plugin-kafka" (master)
- Loaded "bro/apache/metron-bro-plugin-kafka"
+ Installing "zeek/apache/metron-bro-plugin-kafka"........
+ Installed "zeek/apache/metron-bro-plugin-kafka" (master)
+ Loaded "zeek/apache/metron-bro-plugin-kafka"
```
1. Run the following command to ensure that the plugin was installed
successfully.
```
- $ bro -N Apache::Kafka
+ $ zeek -N Apache::Kafka
Apache::Kafka - Writes logs to Kafka (dynamic, version 0.3)
```
### Manual Installation
-Manually installing the plugin should only occur in situations where
installing and configuring `bro-pkg` is not reasonable, such as in a docker
container. If you are running bro in an environment where you do not have
Internet connectivity, investigate
[bundles](https://bro-package-manager.readthedocs.io/en/stable/bro-pkg.html#bundle)
or creating an internal [package
source](https://bro-package-manager.readthedocs.io/en/stable/source.html).
+Manually installing the plugin should *only* occur in situations where
installing and configuring `zkg` is not reasonable. If you are running zeek in
an environment where you do not have Internet connectivity, investigate
[bundles](https://docs.zeek.org/projects/package-manager/en/stable/zkg.html#bundle)
or creating an internal [package
source](https://docs.zeek.org/projects/package-manager/en/stable/source.html).
These instructions could also be helpful if you were interested in
distributing this as a package (such as a deb or rpm).
-1. Install [librdkafka](https://github.com/edenhill/librdkafka), a native
client library for Kafka. This plugin has been tested against the latest
release of librdkafka, which at the time of this writing is v0.11.5.
+1. Install [librdkafka](https://github.com/edenhill/librdkafka), a native
client library for Kafka. This plugin has been tested against librdkafka
v1.4.2-RC3.
In order to use this plugin within a kerberized Kafka environment, you
will also need `libsasl2` installed and will need to pass `--enable-sasl` to
the `configure` script.
```
- $ curl -L https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz |
tar xvz
- $ cd librdkafka-0.11.5/
+ $ curl -L https://github.com/edenhill/librdkafka/archive/v1.4.2-RC3.tar.gz
| tar xvz
+ $ cd librdkafka-1.4.2-RC3/
$ ./configure --enable-sasl
$ make
$ sudo make install
@@ -82,7 +82,7 @@ These instructions could also be helpful if you were
interested in distributing
1. Build the plugin using the following commands.
```
- $ ./configure --bro-dist=$BRO_SRC
+ $ ./configure --zeek-dist=$zeek_dist --with-librdkafka=$librdkafka_root
$ make
$ sudo make install
```
@@ -90,19 +90,21 @@ These instructions could also be helpful if you were
interested in distributing
1. Run the following command to ensure that the plugin was installed
successfully.
```
- $ bro -N Apache::Kafka
+ $ zeek -N Apache::Kafka
Apache::Kafka - Writes logs to Kafka (dynamic, version 0.3)
```
## Activation
-The following examples highlight different ways that the plugin can be used.
Simply add the Bro script language to your `local.bro` file (for example,
`/usr/share/bro/site/local.bro`) as shown to demonstrate the example.
+The following examples highlight different ways that the plugin can be used.
Simply add the Zeek script language to your `local.zeek` file (for example,
`/usr/share/zeek/site/local.zeek`) as shown to demonstrate the example.
+
+In addition to activating the plugin, when running Zeek in a cluster it is
highly recommended to leverage one or more Zeek
[loggers](https://docs.zeek.org/en/v3.1.2/cluster/index.html#logger) as shown
[here](https://docs.zeek.org/en/v3.1.2/configuration/index.html#basic-cluster-configuration)
to separate logging activities from the manager thread.
### Example 1 - Send a list of logs to kafka
-The goal in this example is to send all HTTP and DNS records to a Kafka topic
named `bro`.
+The goal in this example is to send all HTTP and DNS records to a Kafka topic
named `zeek`.
* Any configuration value accepted by librdkafka can be added to the
`kafka_conf` configuration table.
- * The `topic_name` will default to send all records to a single Kafka topic
called 'bro'.
+ * The `topic_name` will default to send all records to a single Kafka topic
called 'zeek'.
* Defining `logs_to_send` will send the HTTP and DNS records to the brokers
specified in your `Kafka::kafka_conf`.
```
@load packages/metron-bro-plugin-kafka/Apache/Kafka
@@ -114,7 +116,7 @@ redef Kafka::kafka_conf = table(
### Example 2 - Send all active logs
-This plugin has the ability send all active logs to the "bro" kafka topic with
the following configuration.
+This plugin has the ability send all active logs to the "zeek" kafka topic
with the following configuration.
```
@load packages/metron-bro-plugin-kafka/Apache/Kafka
@@ -126,23 +128,23 @@ redef Kafka::kafka_conf = table(
### Example 3 - Send all active logs with exclusions
-You can also specify a blacklist of bro logs to ensure they aren't being sent
to kafka regardless of the `Kafka::send_all_active_logs` and
`Kafka::logs_to_send` configurations. In this example, we will send all of the
enabled logs except for the Conn log.
+You can also specify a blacklist of zeek logs to ensure they aren't being sent
to kafka regardless of the `Kafka::send_all_active_logs` and
`Kafka::logs_to_send` configurations. In this example, we will send all of the
enabled logs except for the Conn log.
```
@load packages/metron-bro-plugin-kafka/Apache/Kafka
redef Kafka::send_all_active_logs = T;
redef Kafka::logs_to_exclude = set(Conn::LOG);
-redef Kafka::topic_name = "bro";
+redef Kafka::topic_name = "zeek";
redef Kafka::kafka_conf = table(
["metadata.broker.list"] = "localhost:9092"
);
```
-### Example 4 - Send each bro log to a unique topic
+### Example 4 - Send each zeek log to a unique topic
It is also possible to send each log stream to a uniquely named topic. The
goal in this example is to send all HTTP records to a Kafka topic named `http`
and all DNS records to a separate Kafka topic named `dns`.
* The `topic_name` value must be set to an empty string.
- * The `$path` value of Bro's Log Writer mechanism is used to define the topic
name.
+ * The `$path` value of Zeek's Log Writer mechanism is used to define the
topic name.
* Any configuration value accepted by librdkafka can be added to the
`$config` configuration table.
* Each log writer accepts a separate configuration table.
@@ -151,7 +153,7 @@ It is also possible to send each log stream to a uniquely
named topic. The goal
redef Kafka::topic_name = "";
redef Kafka::tag_json = T;
-event bro_init() &priority=-10
+event zeek_init() &priority=-10
{
# handles HTTP
local http_filter: Log::Filter = [
@@ -177,10 +179,10 @@ event bro_init() &priority=-10
}
```
-### Example 5 - Bro log filtering
+### Example 5 - Zeek log filtering
-You may want to configure bro to filter log messages with certain
characteristics from being sent to your kafka topics. For instance, Metron
currently doesn't support IPv6 source or destination IPs in the default
enrichments, so it may be helpful to filter those log messages from being sent
to kafka (although there are [multiple ways](#notes) to approach this). In
this example we will do that that, and are assuming a somewhat standard bro
kafka plugin configuration, such that:
- * All bro logs are sent to the default `bro` topic.
+You may want to configure zeek to filter log messages with certain
characteristics from being sent to your kafka topics. For instance, Apache
Metron currently doesn't support IPv6 source or destination IPs in the default
enrichments, so it may be helpful to filter those log messages from being sent
to kafka (although there are [multiple ways](#notes) to approach this). In
this example we will do that that, and are assuming a somewhat standard zeek
kafka plugin configuration, such that:
+ * All zeek logs are sent to the default `zeek` topic.
* Each JSON message is tagged with the appropriate log type (such as `http`,
`dns`, or `conn`), by setting `Kafka::tag_json` to true.
* If the log message contains a 128 byte long source or destination IP
address, the log is not sent to kafka.
@@ -188,7 +190,7 @@ You may want to configure bro to filter log messages with
certain characteristic
@load packages/metron-bro-plugin-kafka/Apache/Kafka
redef Kafka::tag_json = T;
-event bro_init() &priority=-10
+event zeek_init() &priority=-10
{
# handles HTTP
Log::add_filter(HTTP::LOG, [
@@ -224,46 +226,45 @@ event bro_init() &priority=-10
#### Notes
* `logs_to_send` is mutually exclusive with `$pred`, thus for each log you
want to set `$pred` on, you must individually setup a `Log::add_filter` and
refrain from including that log in `logs_to_send`.
- * In Bro 2.5.x the bro project introduced a [logger
function](https://www.bro.org/sphinx/cluster/index.html#logger) which removes
the logging functions from the manager thread, and taking advantage of that is
highly recommended. If you are running this plugin on Bro 2.4.x, you may
encounter issues where the manager thread is taking on too much responsibility
and pinning a single CPU core without the ability to spread the load across
additional cores. In this case, it may be in your be [...]
- * You can also filter IPv6 logs from within your Metron cluster [using
Stellar](https://github.com/apache/metron/tree/master/metron-stellar/stellar-common#is_ip).
In that case, you wouldn't apply a predicate in your bro configuration, and
instead Stellar would filter the logs out before they were processed by the
enrichment layer of Metron.
- * It is also possible to use the `is_v6_subnet()` bro function in your
predicate, as of their [2.5
release](https://www.bro.org/sphinx-git/install/release-notes.html#bro-2-5),
however the above example should work on [bro
2.4](https://www.bro.org/sphinx-git/install/release-notes.html#bro-2-4) and
newer, which has been the focus of the kafka plugin.
+ * The
[`is_v6_addr()`](https://docs.zeek.org/en/v3.1.2/scripts/base/bif/zeek.bif.zeek.html#id-is_v6_addr)
function can also be used in your `$pred` to identify if an IP address is IPv6.
+ * Alternatively, if you are using Apache Metron to pull from the specified
kafka topic, you could filter the IPv6 logs [using
Stellar](https://metron.apache.org/current-book/metron-stellar/stellar-common/index.html#IS_IP).
In that case Stellar would filter the logs out and a `$pred` would not be
necessary. The benefit to this approach is that kafka would receive an
unfiltered set of logs.
### Example 6 - Sending a log to multiple topics
-You are able to send a single bro log to multiple different kafka topics in
the same kafka cluster by overriding the default topic (configured with
`Kafka::topic_name`) by creating a custom bro `Log::Filter`. In this example,
the DHCP, RADIUS, and DNS logs are sent to the "bro" topic; the RADIUS log is
duplicated to the "shew_bro_radius" topic; and the DHCP log is duplicated to
the "shew_bro_dhcp" topic.
+You are able to send a single zeek log to multiple different kafka topics in
the same kafka cluster by overriding the default topic (configured with
`Kafka::topic_name`) by creating a custom zeek `Log::Filter`. In this example,
the DHCP, RADIUS, and DNS logs are sent to the "zeek" topic; the RADIUS log is
duplicated to the "shew_zeek_radius" topic; and the DHCP log is duplicated to
the "shew_zeek_dhcp" topic.
```
@load packages/metron-bro-plugin-kafka/Apache/Kafka
redef Kafka::logs_to_send = set(DHCP::LOG, RADIUS::LOG, DNS::LOG);
-redef Kafka::topic_name = "bro";
+redef Kafka::topic_name = "zeek";
redef Kafka::kafka_conf = table(
["metadata.broker.list"] =
"server1.example.com:9092,server2.example.com:9092"
);
redef Kafka::tag_json = T;
-event bro_init() &priority=-10
+event zeek_init() &priority=-10
{
- # Send RADIUS to the shew_bro_radius topic
+ # Send RADIUS to the shew_zeek_radius topic
local shew_radius_filter: Log::Filter = [
$name = "kafka-radius-shew",
$writer = Log::WRITER_KAFKAWRITER,
- $path = "shew_bro_radius"
- $config = table(["topic_name"] = "shew_bro_radius")
+ $path = "shew_zeek_radius"
+ $config = table(["topic_name"] = "shew_zeek_radius")
];
Log::add_filter(RADIUS::LOG, shew_radius_filter);
- # Send DHCP to the shew_bro_dhcp topic
+ # Send DHCP to the shew_zeek_dhcp topic
local shew_dhcp_filter: Log::Filter = [
$name = "kafka-dhcp-shew",
$writer = Log::WRITER_KAFKAWRITER,
- $path = "shew_bro_dhcp"
- $config = table(["topic_name"] = "shew_bro_dhcp")
+ $path = "shew_zeek_dhcp"
+ $config = table(["topic_name"] = "shew_zeek_dhcp")
];
Log::add_filter(DHCP::LOG, shew_dhcp_filter);
}
```
-_Note_: Because `Kafka::tag_json` is set to True in this example, the value
of `$path` is used as the tag for each `Log::Filter`. If you were to add a log
filter with the same `$path` as an existing filter, Bro will append "-N", where
N is an integer starting at 2, to the end of the log path so that each filter
has its own unique log path. For instance, the second instance of `conn` would
become `conn-2`.
+_Note_: Because `Kafka::tag_json` is set to True in this example, the value
of `$path` is used as the tag for each `Log::Filter`. If you were to add a log
filter with the same `$path` as an existing filter, Zeek will append "-N",
where N is an integer starting at 2, to the end of the log path so that each
filter has its own unique log path. For instance, the second instance of `conn`
would become `conn-2`.
## Settings
@@ -294,10 +295,10 @@ redef Kafka::logs_to_exclude = set(Conn::LOG, DNS::LOG);
### `topic_name`
-The name of the topic in Kafka where all Bro logs will be sent to.
+The name of the topic in Kafka where all Zeek logs will be sent to.
```
-redef Kafka::topic_name = "bro";
+redef Kafka::topic_name = "zeek";
```
### `kafka_conf`
@@ -305,12 +306,12 @@ redef Kafka::topic_name = "bro";
The global configuration settings for Kafka. These values are passed through
directly to librdkafka. Any valid librdkafka settings can be defined in this
table. The full set of valid librdkafka settings are available
-[here](https://github.com/edenhill/librdkafka/blob/v0.11.5/CONFIGURATION.md).
+[here](https://github.com/edenhill/librdkafka/blob/v1.4.2-RC3/CONFIGURATION.md).
```
redef Kafka::kafka_conf = table(
["metadata.broker.list"] = "localhost:9092",
- ["client.id"] = "bro"
+ ["client.id"] = "zeek"
);
```
@@ -345,10 +346,12 @@ redef Kafka::max_wait_on_shutdown = 3000;
A comma separated list of debug contexts in librdkafka which you want to
enable. The available contexts are:
+* all
* generic
* broker
* topic
* metadata
+* feature
* queue
* msg
* protocol
@@ -356,7 +359,10 @@ enable. The available contexts are:
* security
* fetch
* feature
-* all
+* interceptor
+* plugin
+* consumer
+* admin
## Kerberos
@@ -376,7 +382,7 @@ builtin.features = gzip,snappy,ssl,sasl,regex
### Producer Config
As stated above, you can configure the producer kafka configs in
-`${BRO_HOME}/share/bro/site/local.bro`. There are a few configs
+`${ZEEK_HOME}/share/zeek/site/local.zeek`. There are a few configs
necessary to set, which are described
[here](https://github.com/edenhill/librdkafka/wiki/Using-SASL-with-librdkafka).
For an environment where the following is true:
@@ -385,19 +391,19 @@ For an environment where the following is true:
* The keytab used is the `metron` keytab
* The service principal for `metron` is `met...@example.com`
-The kafka topic `bro` has been given permission for the `metron` user to
+The kafka topic `zeek` has been given permission for the `metron` user to
write:
```
# login using the metron user
kinit -kt /etc/security/keytabs/metron.headless.keytab met...@example.com
-${KAFKA_HOME}/kafka-broker/bin/kafka-acls.sh --authorizer
kafka.security.auth.SimpleAclAuthorizer --authorizer-properties
zookeeper.connect=node1:2181 --add --allow-principal User:metron --topic bro
+${KAFKA_HOME}/kafka-broker/bin/kafka-acls.sh --authorizer
kafka.security.auth.SimpleAclAuthorizer --authorizer-properties
zookeeper.connect=node1:2181 --add --allow-principal User:metron --topic zeek
```
-The following is how the `${BRO_HOME}/share/bro/site/local.bro` looks:
+The following is how the `${ZEEK_HOME}/share/zeek/site/local.zeek` looks:
```
@load packages/metron-bro-plugin-kafka/Apache/Kafka
redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG);
-redef Kafka::topic_name = "bro";
+redef Kafka::topic_name = "zeek";
redef Kafka::tag_json = T;
redef Kafka::kafka_conf = table( ["metadata.broker.list"] = "node1:6667"
, ["security.protocol"] = "SASL_PLAINTEXT"
diff --git a/bro-pkg.meta b/bro-pkg.meta
deleted file mode 100644
index 86e2334..0000000
--- a/bro-pkg.meta
+++ /dev/null
@@ -1,15 +0,0 @@
-[package]
-description = A Bro log writer plugin that sends logging output to Kafka.
-tags = log writer, bro plugin, kafka
-script_dir = build/scripts/Apache/Kafka
-build_command = ./configure --bro-dist=%(bro_dist)s
--with-librdkafka=%(LIBRDKAFKA_ROOT)s && make
-test_command = ( cd tests && btest -d )
-plugin_dir = build
-version = 0.3
-depends =
- bro >=2.5.0
- bro-pkg >=1.2
-external_depends =
- librdkafka ~0.11.5
-user_vars =
- LIBRDKAFKA_ROOT [/usr/local/lib] "Path to librdkafka installation tree"
diff --git a/configure b/configure
index d053488..0e791d0 100755
--- a/configure
+++ b/configure
@@ -16,6 +16,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+#
+# The upstream version of this is at
+#
https://github.com/zeek/zeek-aux/blob/master/plugin-support/skeleton/configure
+#
# Wrapper for viewing/setting options that the plugin's CMake
# scripts will recognize.
#
@@ -30,23 +34,20 @@ if [ -e `dirname $0`/configure.plugin ]; then
. `dirname $0`/configure.plugin
fi
-# Check for `cmake` command.
-type cmake > /dev/null 2>&1 || {
- echo "\
-This package requires CMake, please install it first, then you may
-use this configure script to access CMake equivalent functionality.\
-" >&2;
- exit 1;
-}
-
usage() {
cat 1>&2 <<EOF
Usage: $0 [OPTIONS]
Plugin Options:
- --bro-dist=DIR Path to Bro source tree
+ --cmake=PATH Path to CMake binary
+ --zeek-dist=DIR Path to Zeek source tree
--install-root=DIR Path where to install plugin into
+ --with-binpac=DIR Path to BinPAC installation root
+ --with-broker=DIR Path to Broker installation root
+ --with-caf=DIR Path to CAF installation root
+ --with-bifcl=PATH Path to bifcl executable
+ --enable-debug Compile in debugging mode
EOF
if type plugin_usage >/dev/null 2>&1; then
@@ -69,7 +70,7 @@ append_cache_entry () {
# set defaults
builddir=build
-brodist=`cd ../../.. && pwd`
+zeekdist=""
installroot="default"
CMakeCacheEntries=""
@@ -83,15 +84,42 @@ while [ $# -ne 0 ]; do
--help|-h)
usage
;;
- --bro-dist=*)
- brodist=`cd $optarg && pwd`
+
+ --cmake=*)
+ CMakeCommand=$optarg
+ ;;
+
+ --zeek-dist=*)
+ zeekdist=`cd $optarg && pwd`
;;
+
--install-root=*)
installroot=$optarg
;;
- --with-openssl=*)
- append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
+
+ --with-binpac=*)
+ append_cache_entry BinPAC_ROOT_DIR PATH $optarg
+ binpac_root=$optarg
;;
+
+ --with-broker=*)
+ append_cache_entry BROKER_ROOT_DIR PATH $optarg
+ broker_root=$optarg
+ ;;
+
+ --with-caf=*)
+ append_cache_entry CAF_ROOT_DIR PATH $optarg
+ caf_root=$optarg
+ ;;
+
+ --with-bifcl=*)
+ append_cache_entry BifCl_EXE PATH $optarg
+ ;;
+
+ --enable-debug)
+ append_cache_entry BRO_PLUGIN_ENABLE_DEBUG BOOL true
+ ;;
+
*)
if type plugin_option >/dev/null 2>&1; then
plugin_option $1 && shift && continue;
@@ -104,13 +132,63 @@ while [ $# -ne 0 ]; do
shift
done
-if [ ! -e "$brodist/bro-path-dev.in" ]; then
- echo "Cannot determine Bro source directory, use --bro-dist=DIR."
- exit 1
+if [ -z "$CMakeCommand" ]; then
+ # prefer cmake3 over "regular" cmake (cmake == cmake2 on RHEL)
+ if command -v cmake3 >/dev/null 2>&1 ; then
+ CMakeCommand="cmake3"
+ elif command -v cmake >/dev/null 2>&1 ; then
+ CMakeCommand="cmake"
+ else
+ echo "This package requires CMake, please install it first."
+ echo "Then you may use this script to configure the CMake build."
+ echo "Note: pass --cmake=PATH to use cmake in non-standard locations."
+ exit 1;
+ fi
fi
-append_cache_entry BRO_DIST PATH $brodist
-append_cache_entry CMAKE_MODULE_PATH PATH $brodist/cmake
+if [ -z "$zeekdist" ]; then
+ if type zeek-config >/dev/null 2>&1; then
+ zeek_config="zeek-config"
+ else
+ echo "Either 'zeek-config' must be in PATH or '--zeek-dist=<path>'
used"
+ exit 1
+ fi
+
+ append_cache_entry BRO_CONFIG_PREFIX PATH `${zeek_config} --prefix`
+ append_cache_entry BRO_CONFIG_INCLUDE_DIR PATH `${zeek_config}
--include_dir`
+ append_cache_entry BRO_CONFIG_PLUGIN_DIR PATH `${zeek_config} --plugin_dir`
+ append_cache_entry BRO_CONFIG_CMAKE_DIR PATH `${zeek_config} --cmake_dir`
+ append_cache_entry CMAKE_MODULE_PATH PATH `${zeek_config} --cmake_dir`
+
+ build_type=`${zeek_config} --build_type`
+
+ if [ "$build_type" = "debug" ]; then
+ append_cache_entry BRO_PLUGIN_ENABLE_DEBUG BOOL true
+ fi
+
+ if [ -z "$binpac_root" ]; then
+ append_cache_entry BinPAC_ROOT_DIR PATH `${zeek_config} --binpac_root`
+ fi
+
+ if [ -z "$broker_root" ]; then
+ append_cache_entry BROKER_ROOT_DIR PATH `${zeek_config} --broker_root`
+ fi
+
+ if [ -z "$caf_root" ]; then
+ append_cache_entry CAF_ROOT_DIR PATH `${zeek_config} --caf_root`
+ fi
+else
+ if [ ! -e "$zeekdist/zeek-path-dev.in" ]; then
+ echo "$zeekdist does not appear to be a valid Zeek source tree."
+ exit 1
+ fi
+
+ # BRO_DIST is the canonical/historical name used by plugin CMake scripts
+ # ZEEK_DIST doesn't serve a function at the moment, but set/provided anyway
+ append_cache_entry BRO_DIST PATH $zeekdist
+ append_cache_entry ZEEK_DIST PATH $zeekdist
+ append_cache_entry CMAKE_MODULE_PATH PATH $zeekdist/cmake
+fi
if [ "$installroot" != "default" ]; then
mkdir -p $installroot
@@ -118,12 +196,12 @@ if [ "$installroot" != "default" ]; then
fi
echo "Build Directory : $builddir"
-echo "Bro Source Directory : $brodist"
+echo "Zeek Source Directory : $zeekdist"
mkdir -p $builddir
cd $builddir
-cmake $CMakeCacheEntries ..
+"$CMakeCommand" $CMakeCacheEntries ..
echo "# This is the command used to configure this build" > config.status
echo $command >> config.status
diff --git a/configure.plugin b/configure.plugin
index c7e6662..7273ed0 100644
--- a/configure.plugin
+++ b/configure.plugin
@@ -16,6 +16,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+#
# Hooks to add custom options to the configure script.
#
@@ -31,13 +32,15 @@ plugin_option()
{
case "$1" in
--with-librdkafka=*)
- append_cache_entry LibRDKafka_ROOT_DIR PATH $optarg
+ append_cache_entry LibRDKafka_ROOT_DIR PATH "${optarg}"
+ return 0
;;
--with-openssl=*)
- append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
+ append_cache_entry OpenSSL_ROOT_DIR PATH "${optarg}"
+ return 0
;;
*)
return 1;
- ;;
+ ;;
esac
}
diff --git a/dev_utilities/release-utils/metron-bro-kafka-rc-check
b/dev_utilities/release-utils/metron-bro-kafka-rc-check
index edecaca..279255d 100755
--- a/dev_utilities/release-utils/metron-bro-kafka-rc-check
+++ b/dev_utilities/release-utils/metron-bro-kafka-rc-check
@@ -20,8 +20,8 @@ shopt -s nocasematch
function help {
echo " "
echo "usage: ${0}"
- echo " -v/--version=<version> The version of the metron bro plugin
kafka release. [Required]"
- echo " -c/--candidate=<RC#> Defines the Release Candidate.
[Required]"
+ echo " -v/--version=<version> The version of the
metron-bro-plugin-kafka release. [Required]"
+ echo " -c/--candidate=<RC#> Defines the Release Candidate. [Required]"
echo " -h/--help Usage information."
echo " "
echo "example: "
@@ -34,9 +34,9 @@
METRON_DIST=${APACHE_REPO}"dev/metron/metron-bro-plugin-kafka/"
METRON_KEYS=${APACHE_REPO}"release/metron/KEYS"
#
-# runs the bro kafka plugin's docker based tests
+# runs the package kafka plugin's docker based tests
#
-function run_bro_docker {
+function run_package_docker {
cd docker &> /dev/null || { echo "failed to change directory to docker" ;
exit 1; }
./run_end_to_end.sh
@@ -48,9 +48,9 @@ function run_bro_docker {
}
#
-# runs the finish bro docker script to cleanup
+# runs the finish package docker script to cleanup
#
-function finish_bro_docker {
+function finish_package_docker {
cd docker &> /dev/null || { echo "failed to change directory to docker";
exit 1; }
./finish_end_to_end.sh
@@ -119,7 +119,7 @@ fi
if [[ "$VERSION" =~ ^[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2} ]]; then
PLUGIN_VERSION="$VERSION"
else
- echo "[ERROR] "$VERSION" may not be a valid version number"
+ echo "[ERROR] \"$VERSION\" may not be a valid version number"
exit 1
fi
@@ -139,11 +139,11 @@ else
exit 1
fi
-echo "Metron Bro Plugin Kafka Version $PLUGIN_VERSION"
+echo "metron-bro-plugin-kafka Version $PLUGIN_VERSION"
echo "Release Candidate $RC"
PLUGIN_RC_DIST="$METRON_DIST$PLUGIN_VERSION-$UPPER_RC"
-echo "Metron Bro Plugin Kafka RC Distribution Root is $PLUGIN_RC_DIST"
+echo "metron-bro-plugin-kafka RC Distribution Root is $PLUGIN_RC_DIST"
# working directory
WORK="$HOME/tmp/metron-bro-plugin-kafka_$PLUGIN_VERSION-$RC"
@@ -192,15 +192,15 @@ if ! gpg --import KEYS ; then
exit 1
fi
-echo "Verifying Metron Bro Plugin Kafka Assembly"
+echo "Verifying metron-bro-plugin-kafka Assembly"
if ! gpg --verify
./"apache-metron-bro-plugin-kafka_$PLUGIN_VERSION-$RC.tar.gz.asc"
"apache-metron-bro-plugin-kafka_$PLUGIN_VERSION-$RC.tar.gz" ; then
- echo "[ERROR] failed to verify Metron Bro Plugin Kafka Assembly"
+ echo "[ERROR] failed to verify metron-bro-plugin-kafka Assembly"
exit 1
fi
echo "Unpacking Assemblies"
if ! tar -xzf "apache-metron-bro-plugin-kafka_$PLUGIN_VERSION-$RC.tar.gz" ;
then
- echo "[ERROR] failed to unpack Metron Bro Plugin Kafka Assembly"
+ echo "[ERROR] failed to unpack metron-bro-plugin-kafka Assembly"
exit 1
fi
@@ -209,12 +209,12 @@ echo ""
read -p " run test suite [yN] " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
- echo " please verify that no bro docker containers are running before
continuing,"
- read -p " no bro docker containers are running, ready to proceed [yN]
" -n 1 -r
+ echo " please verify that no metron-bro-plugin-kafka docker containers are
running before continuing,"
+ read -p " no metron-bro-plugin-kafka docker containers are running,
ready to proceed [yN] " -n 1 -r
if [[ $REPLY =~ ^[Yy]$ ]]; then
- cd apache-metron-bro-plugin-kafka_$PLUGIN_VERSION-$RC || exit 1
- run_bro_docker
- finish_bro_docker
+ cd "apache-metron-bro-plugin-kafka_$PLUGIN_VERSION-$RC" || exit 1
+ run_package_docker
+ finish_package_docker
else
echo " when you are ready and the containers are stopped, please
cd into the docker"
echo " directory and execute the run_end_to_end.sh script"
diff --git a/docker/README.md b/docker/README.md
index 3653181..4155d6b 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -15,7 +15,7 @@
## Docker support for testing metron-bro-plugin-kafka
-These scripts and containers provide support for building and testing Bro and
the metron-bro-plugin-kafka using a number of Docker containers.
+These scripts and containers provide support for building and testing Zeek and
the metron-bro-plugin-kafka using a number of Docker containers.
The use of these scripts and containers allow an easier, automated workflow
for testing new features, fixes, or regressions than before.
One of the goals is for this to be extensible, such that new scripts can be
introduced and run as well. This will allow, for example, one or more
testing scripts to be added to a pull request, and subsequently to a test
suite.
@@ -25,7 +25,7 @@ testing scripts to be added to a pull request, and
subsequently to a test suite.
```bash
├── containers
-│ └── bro
+│ └── zeek
│ └── kafka
│ └── zookeeper
├── data
@@ -34,38 +34,38 @@ testing scripts to be added to a pull request, and
subsequently to a test suite.
└── test_output
```
- `containers`: The parent of all of the containers that this project defines.
We use several containers, not all of them ours.
- - `bro`: The directory for our bro container, used for building bro, the
librdkafka, and our plugin, as well as running bro.
+ - `zeek`: The directory for our zeek container, used for building zeek, the
librdkafka, and our plugin, as well as running zeek.
- `kafka`: The directory for our kafka container.
- `zookeeper`: The directory for our zookeeper container.
- `data`: The default path for pcap data to be used in tests.
-- `in_docker_scripts`: This directory is mapped to the bro docker container as
/root/built_in_scripts. These represent the library of scripts we provide to
be run in the docker container.
+- `in_docker_scripts`: This directory is mapped to the zeek docker container
as /root/built_in_scripts. These represent the library of scripts we provide
to be run in the docker container.
- `scripts`: These are the scripts that are run on the host for creating the
docker bits, running containers, running or executing commands against
containers ( such as executing one of the built_in_scripts ), and cleaning up
resources.
-- `test_output`: Directory where the bro logs and kafka logs per test/pcap are
stored.
+- `test_output`: Directory where the zeek logs and kafka logs per test/pcap
are stored.
#### Scripts that execute _in_ the docker container
```bash
-├── build_bro_plugin.sh
-├── configure_bro_plugin.sh
+├── build_zeek_plugin.sh
+├── configure_zeek_plugin.sh
├── process_data_file.sh
```
-- `build_bro_plugin.sh`: Runs `bro-pkg` to build and install the provided
version of the plugin.
-- `configure_bro_plugin.sh`: Configures the plugin for the kafka container,
and routes all traffic types.
+- `build_zeek_plugin.sh`: Runs `zeek-pkg` to build and install the provided
version of the plugin.
+- `configure_zeek_plugin.sh`: Configures the plugin for the kafka container,
and routes all traffic types.
###### Parameters
```bash
- --kafka-topic [OPTIONAL] The kafka topic to configure.
Default: bro"
+ --kafka-topic [OPTIONAL] The kafka topic to configure.
Default: zeek"
```
-- `process_data_file.sh`: Runs `bro -r` on the passed file
+- `process_data_file.sh`: Runs `zeek -r` on the passed file
#### Scripts executed on the host to setup and interact with the docker
containers
```bash
├── analyze_results.sh
-├── docker_execute_build_bro_plugin.sh
-├── docker_execute_configure_bro_plugin.sh
+├── docker_execute_build_zeek_plugin.sh
+├── docker_execute_configure_zeek_plugin.sh
├── docker_execute_create_topic_in_kafka.sh
├── docker_execute_process_data_file.sh
├── docker_execute_shell.sh
@@ -81,51 +81,51 @@ testing scripts to be added to a pull request, and
subsequently to a test suite.
```bash
--test-directory [REQUIRED] The directory for the tests
```
-- `docker_execute_build_bro_plugin.sh`: Executes `build_bro_plugin.sh` in the
bro container
+- `docker_execute_build_zeek_plugin.sh`: Executes `build_zeek_plugin.sh` in
the zeek container
###### Parameters
```bash
- --container-name [OPTIONAL] The Docker container name.
Default: metron-bro-plugin-kafka_bro_1
+ --container-name [OPTIONAL] The Docker container name.
Default: metron-bro-plugin-kafka_zeek_1
```
-- `docker_execute_configure_bro_plugin.sh`: Executes `configure_bro_plugin.sh`
in the bro container
+- `docker_execute_configure_zeek_plugin.sh`: Executes
`configure_zeek_plugin.sh` in the zeek container
###### Parameters
```bash
- --container-name [OPTIONAL] The Docker container name.
Default: metron-bro-plugin-kafka_bro_1
+ --container-name [OPTIONAL] The Docker container name.
Default: metron-bro-plugin-kafka_zeek_1
```
- `docker_execute_create_topic_in_kafka.sh`: Creates the specified kafka topic
in the kafka container
###### Parameters
```bash
--container-name [OPTIONAL] The Docker container name.
Default: metron-bro-plugin-kafka_kafka_1
- --kafka-topic [OPTIONAL] The kafka topic to create.
Default: bro
+ --kafka-topic [OPTIONAL] The kafka topic to create.
Default: zeek
```
-- `docker_execute_process_data_file.sh`: Executes `process_data_file.sh` in
the bro container
+- `docker_execute_process_data_file.sh`: Executes `process_data_file.sh` in
the zeek container
###### Parameters
```bash
- --container-name [OPTIONAL] The Docker container name.
Default: metron-bro-plugin-kafka_bro_1
+ --container-name [OPTIONAL] The Docker container name.
Default: metron-bro-plugin-kafka_zeek_1
```
- `docker_execute_shell.sh`: `docker execute -i -t bash` to get a shell in a
given container
###### Parameters
```bash
- --container-name [OPTIONAL] The Docker container name.
Default: metron-bro-plugin-kafka_bro_1
+ --container-name [OPTIONAL] The Docker container name.
Default: metron-bro-plugin-kafka_zeek_1
```
- `docker_run_consume_kafka.sh`: Runs an instance of the kafka container, with
the console consumer `kafka-console-consumer.sh --topic $KAFKA_TOPIC --offset
$OFFSET --partition 0 --bootstrap-server kafka:9092`
###### Parameters
```bash
--network-name [OPTIONAL] The Docker network name. Default:
metron-bro-plugin-kafka_default
--offset [OPTIONAL] The kafka offset. Default: 0
- --kafka-topic [OPTIONAL] The kafka topic to consume from.
Default: bro
+ --kafka-topic [OPTIONAL] The kafka topic to consume from.
Default: zeek
```
- `docker_run_get_offset_kafka.sh`: Runs an instance of the kafka container
and gets the current offset for the specified topic
###### Parameters
```bash
--network-name [OPTIONAL] The Docker network name. Default:
metron-bro-plugin-kafka_default
- --kafka-topic [OPTIONAL] The kafka topic to get the offset
from. Default: bro
+ --kafka-topic [OPTIONAL] The kafka topic to get the offset
from. Default: zeek
```
- `download_sample_pcaps.sh`: Downloads the sample pcaps to a specified
directory. If they exist, it is a no-op
> The sample pcaps are:
- > -
https://github.com/bro/try-bro/blob/master/manager/static/pcaps/exercise_traffic.pcap
+ > -
https://github.com/zeek/try-zeek/blob/master/manager/static/pcaps/exercise_traffic.pcap
> -
http://downloads.digitalcorpora.org/corpora/network-packet-dumps/2008-nitroba/nitroba.pcap
- > -
https://github.com/bro/try-bro/raw/master/manager/static/pcaps/ssh.pcap
+ > -
https://github.com/zeek/try-zeek/raw/master/manager/static/pcaps/ssh.pcap
> -
https://github.com/markofu/pcaps/blob/master/PracticalPacketAnalysis/ppa-capture-files/ftp.pcap?raw=true
> -
https://github.com/EmpowerSecurityAcademy/wireshark/blob/master/radius_localhost.pcapng?raw=true
> - https://github.com/kholia/my-pcaps/blob/master/VNC/07-vnc
@@ -153,12 +153,12 @@ This script does the following:
1. Runs docker compose
1. Creates the specified topic
1. Downloads sample PCAP data
-1. Runs the bro container in the background
-1. Builds the bro plugin
-1. Configures the bro plugin
-1. Runs bro against all the pcap data, one at a time
-1. Executes a kafka client to read the data from bro for each pcap file
-1. Stores the output kafka messages and the bro logs into the test_output
directory
+1. Runs the zeek container in the background
+1. Builds the zeek plugin
+1. Configures the zeek plugin
+1. Runs zeek against all the pcap data, one at a time
+1. Executes a kafka client to read the data from zeek for each pcap file
+1. Stores the output kafka messages and the zeek logs into the test_output
directory
```bash
>tree Tue_Jan__8_21_54_10_EST_2019
Tue_Jan__8_21_54_10_EST_2019
@@ -171,7 +171,6 @@ This script does the following:
│ ├── http.log
│ ├── kafka-output.log
│ ├── known_certs.log
- │ ├── known_devices.log
│ ├── loaded_scripts.log
│ ├── notice.log
│ ├── packet_filter.log
@@ -194,10 +193,10 @@ This script does the following:
│ ├── software.log
│ └── stats.log
```
-1. Creates a results.csv for each pcap that has the line counts of the kafka
and the bro output for each log
+1. Creates a results.csv for each pcap that has the line counts of the kafka
and the zeek output for each log
1. Prints all the results.csv to the screen
-As we can see, the output is a folder named for the test run time, with a sub
folder per pcap, containing all the bro logs and the `kafka_output.log`.
+As we can see, the output is a folder named for the test run time, with a sub
folder per pcap, containing all the zeek logs and the `kafka_output.log`.
At this point the containers are up and running in the background.
@@ -207,7 +206,7 @@ Other scripts may then be used to do your testing, for
example running:
./scripts/docker_execute_shell.sh
```
-> NOTE: If the scripts are run repeatedly, and there is no change in bro or
the librdkafka, the line `./run_end_to_end.sh` can be replaced by
`./run_end_to_end.sh --skip-docker-build`, which uses the `--skip-docker-build`
flag to not rebuild the containers, saving the significant time of rebuilding
bro and librdkafka.
+> NOTE: If the scripts are run repeatedly, and there is no change in zeek or
the librdkafka, the line `./run_end_to_end.sh` can be replaced by
`./run_end_to_end.sh --skip-docker-build`, which uses the `--skip-docker-build`
flag to not rebuild the containers, saving the significant time of rebuilding
zeek and librdkafka.
> NOTE: After you are done, you must call the `finish_end_to_end.sh` script to
> cleanup.
@@ -215,12 +214,12 @@ Other scripts may then be used to do your testing, for
example running:
##### `run_end_to_end.sh`
###### Parameters
```bash
---skip-docker-build [OPTIONAL] Skip build of bro docker machine.
+--skip-docker-build [OPTIONAL] Skip build of zeek docker machine.
--no-pcaps [OPTIONAL] Do not run pcaps.
--data-path [OPTIONAL] The pcap data path. Default: ./data
---kafka-topic [OPTIONAL] The kafka topic name to use.
Default: bro
+--kafka-topic [OPTIONAL] The kafka topic name to use.
Default: zeek
--plugin-version [OPTIONAL] The plugin version. Default: the
current branch name
```
-> NOTE: The provided `--plugin-version` is passed to the [`bro-pkg
install`](https://docs.zeek.org/projects/package-manager/en/stable/bro-pkg.html#install-command)
command within the container, which allows you to specify a version tag,
branch name, or commit hash. However, that tag, branch, or commit *must* be
available in the currently checked out plugin repository.
+> NOTE: The provided `--plugin-version` is passed to the [`zeek-pkg
install`](https://docs.zeek.org/projects/package-manager/en/stable/zeek-pkg.html#install-command)
command within the container, which allows you to specify a version tag,
branch name, or commit hash. However, that tag, branch, or commit *must* be
available in the currently checked out plugin repository.
diff --git a/docker/containers/bro/requirements-to-freeze.txt
b/docker/containers/bro/requirements-to-freeze.txt
deleted file mode 100644
index 04cb725..0000000
--- a/docker/containers/bro/requirements-to-freeze.txt
+++ /dev/null
@@ -1 +0,0 @@
-bro-pkg
diff --git a/docker/containers/bro/.screenrc b/docker/containers/zeek/.screenrc
similarity index 100%
rename from docker/containers/bro/.screenrc
rename to docker/containers/zeek/.screenrc
diff --git a/docker/containers/bro/Dockerfile
b/docker/containers/zeek/Dockerfile
similarity index 67%
rename from docker/containers/bro/Dockerfile
rename to docker/containers/zeek/Dockerfile
index 3de49e6..51014e4 100644
--- a/docker/containers/bro/Dockerfile
+++ b/docker/containers/zeek/Dockerfile
@@ -14,44 +14,50 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM centos:7
-ARG BRO_VERSION
+FROM centos:8
+ARG ZEEK_VERSION
ARG LIBRDKAFKA_VERSION
-# install development tools
-RUN yum -y groupinstall "Development Tools" && \
- yum -y install cmake make gcc gcc-c++ \
+# install powertools for libpcap-devel
+RUN dnf install -y 'dnf-command(config-manager)' && \
+ yum config-manager --set-enabled PowerTools
+
+# install epel for screen
+RUN dnf install -y epel-release
+
+# install prereqs then clean dnf cache
+RUN dnf -y update && \
+ dnf -y install cmake make gcc gcc-c++ \
flex bison libpcap libpcap-devel \
- openssl-devel python-devel swig \
- zlib-devel perl \
+ openssl-devel python3 platform-python-devel \
+ swig zlib-devel perl \
cyrus-sasl cyrus-sasl-devel cyrus-sasl-gssapi \
- git jq screen tree vim
+ git jq screen tree vim && \
+ dnf -y clean all
# copy in the .screenrc
COPY .screenrc /root
-# install bro
+# install zeek
WORKDIR /root
-RUN git clone https://github.com/bro/bro
-WORKDIR bro/
-RUN git checkout "v${BRO_VERSION}"
+RUN git clone https://github.com/zeek/zeek
+WORKDIR zeek/
+RUN git checkout "v${ZEEK_VERSION}"
RUN git submodule update --init --recursive
RUN ./configure
RUN make
RUN make install
-ENV PATH="${PATH}:/usr/local/bro/bin"
+ENV PATH="${PATH}:/usr/local/zeek/bin"
ENV PATH="${PATH}:/usr/bin"
-# install pip3 and bro-pkg
+# install pip3 and zkg
WORKDIR /root
COPY requirements.txt requirements.txt
-RUN yum -y update && \
- yum -y install epel-release && \
- yum -y install python3-pip && \
- yum clean all && \
+RUN dnf -y install python3-pip && \
+ dnf clean all && \
python3 -m pip install --upgrade pip && \
python3 -m pip install -r requirements.txt && \
- bro-pkg autoconfig
+ zkg autoconfig
# install librdkafka
WORKDIR /root
diff --git a/docker/containers/bro/Makefile b/docker/containers/zeek/Makefile
similarity index 100%
rename from docker/containers/bro/Makefile
rename to docker/containers/zeek/Makefile
diff --git a/docker/containers/zeek/requirements-to-freeze.txt
b/docker/containers/zeek/requirements-to-freeze.txt
new file mode 100644
index 0000000..d180205
--- /dev/null
+++ b/docker/containers/zeek/requirements-to-freeze.txt
@@ -0,0 +1 @@
+zkg
diff --git a/docker/containers/bro/requirements.txt
b/docker/containers/zeek/requirements.txt
similarity index 65%
rename from docker/containers/bro/requirements.txt
rename to docker/containers/zeek/requirements.txt
index 6303830..21d0054 100644
--- a/docker/containers/bro/requirements.txt
+++ b/docker/containers/zeek/requirements.txt
@@ -1,7 +1,7 @@
-bro-pkg==2.1.2
btest==0.61
configparser==5.0.0
gitdb==4.0.4
GitPython==3.1.1
-semantic-version==2.8.4
+semantic-version==2.8.5
smmap==3.0.2
+zkg==2.1.2
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 88b6b71..8e54926 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -15,13 +15,13 @@ services:
- ZOOKEEPER_IP=zookeeper
ports:
- "9092:9092"
- bro:
+ zeek:
build:
- context: containers/bro
+ context: containers/zeek
args:
- BRO_VERSION: "2.5.5"
- LIBRDKAFKA_VERSION: "0.11.5"
- image: metron-bro-plugin-kafka_bro:latest
+ ZEEK_VERSION: "3.1.2"
+ LIBRDKAFKA_VERSION: "1.4.2-RC3"
+ image: metron-bro-plugin-kafka_zeek:latest
depends_on:
zookeeper:
condition: service_healthy
diff --git a/docker/in_docker_scripts/build_bro_plugin.sh
b/docker/in_docker_scripts/build_plugin.sh
similarity index 70%
rename from docker/in_docker_scripts/build_bro_plugin.sh
rename to docker/in_docker_scripts/build_plugin.sh
index 401b2a5..b6cf9f7 100755
--- a/docker/in_docker_scripts/build_bro_plugin.sh
+++ b/docker/in_docker_scripts/build_plugin.sh
@@ -21,12 +21,12 @@ shopt -s nocasematch
shopt -s globstar nullglob
shopt -s nocasematch
set -u # nounset
-set -e # errexit
+# set -e (errexit) omitted to enable printfiles function call
set -E # errtrap
set -o pipefail
#
-# Runs bro-pkg to build and install the plugin
+# Runs zkg to build and install the plugin
#
function help {
@@ -38,6 +38,19 @@ function help {
echo " "
}
+function printfiles {
+ echo "==================================================="
+ echo "ERR"
+ cat /root/.zkg/testing/code/clones/code/zkg.test_command.stderr
+ echo "==================================================="
+ echo "OUT"
+ cat /root/.zkg/testing/code/clones/code/zkg.test_command.stdout
+ echo "==================================================="
+ echo ""
+ echo "==================================================="
+ echo ""
+}
+
PLUGIN_VERSION=
# Handle command line options
@@ -84,23 +97,21 @@ cd /root || exit 1
echo "==================================================="
-bro-pkg -vvv install code --version "${PLUGIN_VERSION}" --force
+zkg -vvv test code
rc=$?; if [[ ${rc} != 0 ]]; then
- echo "ERROR running bro-pkg install ${rc}"
+ echo "ERROR running zkg test ${rc}"
+ printfiles
+ exit ${rc}
+fi
+
+zkg -vvv install code --skiptests --version "${PLUGIN_VERSION}" --force
+rc=$?; if [[ ${rc} != 0 ]]; then
+ echo "ERROR running zkg install ${rc}"
+ printfiles
exit ${rc}
fi
-echo "==================================================="
-echo "ERR"
-cat /root/.zkg/testing/code/clones/code/zkg.test_command.stderr
-echo "==================================================="
-echo "OUT"
-cat /root/.zkg/testing/code/clones/code/zkg.test_command.stdout
-echo "==================================================="
-echo ""
-echo "==================================================="
-echo ""
-bro -NN Apache::Kafka
+zeek -NN Apache::Kafka
echo "==================================================="
echo ""
diff --git a/docker/in_docker_scripts/configure_bro_plugin.sh
b/docker/in_docker_scripts/configure_plugin.sh
similarity index 77%
rename from docker/in_docker_scripts/configure_bro_plugin.sh
rename to docker/in_docker_scripts/configure_plugin.sh
index 2b3235a..1c79f14 100755
--- a/docker/in_docker_scripts/configure_bro_plugin.sh
+++ b/docker/in_docker_scripts/configure_plugin.sh
@@ -20,7 +20,7 @@
shopt -s nocasematch
#
-# Configures the bro kafka plugin
+# Configures the zeek kafka plugin
# Configures the kafka broker
# Configures the plugin for all the traffic types
#
@@ -28,13 +28,13 @@ shopt -s nocasematch
function help {
echo " "
echo "usage: ${0}"
- echo " --kafka-topic [OPTIONAL] The kafka topic to
configure. Default: bro"
+ echo " --kafka-topic [OPTIONAL] The kafka topic to
configure. Default: zeek"
echo " -h/--help Usage information."
echo " "
echo " "
}
-KAFKA_TOPIC=bro
+KAFKA_TOPIC=zeek
# Handle command line options
for i in "$@"; do
@@ -70,20 +70,16 @@ done
echo "Configuring kafka plugin"
{
echo "@load packages"
- echo "redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG, Conn::LOG,
DPD::LOG, FTP::LOG, Files::LOG, Known::CERTS_LOG, SMTP::LOG, SSL::LOG,
Weird::LOG, Notice::LOG, DHCP::LOG, SSH::LOG, Software::LOG, RADIUS::LOG,
X509::LOG, Known::DEVICES_LOG, RFB::LOG, Stats::LOG, CaptureLoss::LOG,
SIP::LOG);"
+ echo "redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG, Conn::LOG,
DPD::LOG, FTP::LOG, Files::LOG, Known::CERTS_LOG, SMTP::LOG, SSL::LOG,
Weird::LOG, Notice::LOG, DHCP::LOG, SSH::LOG, Software::LOG, RADIUS::LOG,
X509::LOG, RFB::LOG, Stats::LOG, CaptureLoss::LOG, SIP::LOG);"
echo "redef Kafka::topic_name = \"${KAFKA_TOPIC}\";"
echo "redef Kafka::tag_json = T;"
echo "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] =
\"kafka:9092\");"
echo "redef Kafka::logs_to_exclude = set(Conn::LOG, DHCP::LOG);"
echo "redef Known::cert_tracking = ALL_HOSTS;"
echo "redef Software::asset_tracking = ALL_HOSTS;"
-} >> /usr/local/bro/share/bro/site/local.bro
+} >> /usr/local/zeek/share/zeek/site/local.zeek
-# Load "known-devices-and-hostnames.bro" which is necessary in bro 2.5.5 to
-# create the log Known::DEVICES_LOG
-sed -i '86 a @load policy/protocols/dhcp/known-devices-and-hostnames.bro'
/usr/local/bro/share/bro/site/local.bro
-
-# Comment out the load statement for "log-hostcerts-only.bro" in bro 2.5.5's
-# default local.bro in order to log all certificates to x509.log
-sed -i 's%^@load protocols/ssl/log-hostcerts-only%#&%'
/usr/local/bro/share/bro/site/local.bro
+# Comment out the load statement for "log-hostcerts-only.zeek" in zeek's
+# default local.zeek as of 3.1.2 in order to log all certificates to x509.log
+sed -i 's%^@load protocols/ssl/log-hostcerts-only%#&%'
/usr/local/zeek/share/zeek/site/local.zeek
diff --git a/docker/in_docker_scripts/process_data_file.sh
b/docker/in_docker_scripts/process_data_file.sh
index 4ddfdef..584f0a4 100755
--- a/docker/in_docker_scripts/process_data_file.sh
+++ b/docker/in_docker_scripts/process_data_file.sh
@@ -74,7 +74,7 @@ if [ ! -d /root/data ]; then
exit 1
fi
cd /root/test_output/"${OUTPUT_DIRECTORY_NAME}" || exit 1
-find /root/data -type f -name "${PCAP_FILE_NAME}" -print0 | xargs -0 bro
/usr/local/bro/share/bro/site/local.bro -C -r
+find /root/data -type f -name "${PCAP_FILE_NAME}" -print0 | xargs -0 zeek
/usr/local/zeek/share/zeek/site/local.zeek -C -r
rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc}
fi
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
index ec5f7b0..cb0e24c 100755
--- a/docker/run_end_to_end.sh
+++ b/docker/run_end_to_end.sh
@@ -25,9 +25,9 @@ set -o pipefail
function help {
echo " "
echo "USAGE"
- echo " --skip-docker-build [OPTIONAL] Skip build of bro
docker machine."
+ echo " --skip-docker-build [OPTIONAL] Skip build of zeek
docker machine."
echo " --data-path [OPTIONAL] The pcap data path.
Default: ./data"
- echo " --kafka-topic [OPTIONAL] The kafka topic to
consume from. Default: bro"
+ echo " --kafka-topic [OPTIONAL] The kafka topic to
consume from. Default: zeek"
echo " --plugin-version [OPTIONAL] The plugin version.
Default: the current branch name"
echo " --no-pcap [OPTIONAL] Do not run pcaps."
echo " -h/--help Usage information."
@@ -44,7 +44,7 @@ if (( BASH_VERSINFO[0] < 4 )); then
exit 1
fi
-SKIP_REBUILD_BRO=false
+SKIP_REBUILD_ZEEK=false
NO_PCAP=false
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
PLUGIN_ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. > /dev/null &&
pwd)"
@@ -53,17 +53,17 @@ DATA_PATH="${ROOT_DIR}"/data
DATE=$(date)
LOG_DATE=${DATE// /_}
TEST_OUTPUT_PATH="${ROOT_DIR}/test_output/"${LOG_DATE//:/_}
-KAFKA_TOPIC="bro"
+KAFKA_TOPIC="zeek"
PROJECT_NAME="metron-bro-plugin-kafka"
OUR_SCRIPTS_PATH="${PLUGIN_ROOT_DIR}/docker/in_docker_scripts"
cd "${PLUGIN_ROOT_DIR}" || { echo "NO PLUGIN ROOT" ; exit 1; }
# we may not be checked out from git, check and make it so that we are since
-# bro-pkg requires it
+# zkg requires it
git status &>/dev/null
rc=$?; if [[ ${rc} != 0 ]]; then
- echo "bro-pkg requires the plugin to be a git repo, creating..."
+ echo "zkg requires the plugin to be a git repo, creating..."
git init .
rc=$?; if [[ ${rc} != 0 ]]; then
echo "ERROR> FAILED TO INITIALIZE GIT IN PLUGIN DIRECTORY. ${rc}"
@@ -91,12 +91,12 @@ PLUGIN_VERSION=$(git rev-parse --symbolic-full-name
--abbrev-ref HEAD)
for i in "$@"; do
case $i in
#
- # SKIP_REBUILD_BRO
+ # SKIP_REBUILD_ZEEK
#
# --skip-docker-build
#
--skip-docker-build)
- SKIP_REBUILD_BRO=true
+ SKIP_REBUILD_ZEEK=true
shift # past argument
;;
#
@@ -150,14 +150,14 @@ done
cd "${ROOT_DIR}" || { echo "NO ROOT" ; exit 1; }
echo "Running docker compose with "
-echo "SKIP_REBUILD_BRO = ${SKIP_REBUILD_BRO}"
-echo "DATA_PATH = ${DATA_PATH}"
-echo "KAFKA_TOPIC = ${KAFKA_TOPIC}"
-echo "PLUGIN_VERSION = ${PLUGIN_VERSION}"
+echo "SKIP_REBUILD_ZEEK = ${SKIP_REBUILD_ZEEK}"
+echo "DATA_PATH = ${DATA_PATH}"
+echo "KAFKA_TOPIC = ${KAFKA_TOPIC}"
+echo "PLUGIN_VERSION = ${PLUGIN_VERSION}"
echo "==================================================="
# Run docker compose, rebuilding as specified
-if [[ "$SKIP_REBUILD_BRO" = false ]]; then
+if [[ "$SKIP_REBUILD_ZEEK" = false ]]; then
COMPOSE_PROJECT_NAME="${PROJECT_NAME}" \
DATA_PATH=${DATA_PATH} \
TEST_OUTPUT_PATH=${TEST_OUTPUT_PATH} \
@@ -190,15 +190,15 @@ bash "${SCRIPT_DIR}"/download_sample_pcaps.sh
--data-path="${DATA_PATH}"
# By not catching $? here we are accepting that a failed pcap download will not
# exit the script
-# Build the bro plugin
-bash "${SCRIPT_DIR}"/docker_execute_build_bro_plugin.sh
--plugin-version="${PLUGIN_VERSION}"
+# Build the zeek plugin
+bash "${SCRIPT_DIR}"/docker_execute_build_plugin.sh
--plugin-version="${PLUGIN_VERSION}"
rc=$?; if [[ ${rc} != 0 ]]; then
echo "ERROR> FAILED TO BUILD PLUGIN. CHECK LOGS ${rc}"
exit ${rc}
fi
-# Configure the bro plugin
-bash "${SCRIPT_DIR}"/docker_execute_configure_bro_plugin.sh
--kafka-topic="${KAFKA_TOPIC}"
+# Configure the plugin
+bash "${SCRIPT_DIR}"/docker_execute_configure_plugin.sh
--kafka-topic="${KAFKA_TOPIC}"
rc=$?; if [[ ${rc} != 0 ]]; then
echo "ERROR> FAILED TO CONFIGURE PLUGIN. CHECK LOGS ${rc}"
exit ${rc}
@@ -206,7 +206,7 @@ fi
if [[ "$NO_PCAP" = false ]]; then
# for each pcap in the data directory, we want to
- # run bro then read the output from kafka
+ # run zeek then read the output from kafka
# and output both of them to the same directory named
# for the date/pcap
@@ -258,5 +258,5 @@ if [[ "$NO_PCAP" = false ]]; then
fi
echo ""
echo "Run complete"
-echo "The kafka and bro output can be found at ${TEST_OUTPUT_PATH}"
+echo "The kafka and zeek output can be found at ${TEST_OUTPUT_PATH}"
echo "You may now work with the containers if you will. You need to call
finish_end_to_end.sh when you are done"
diff --git a/docker/scripts/analyze_results.sh
b/docker/scripts/analyze_results.sh
index 01c026f..c106a3a 100755
--- a/docker/scripts/analyze_results.sh
+++ b/docker/scripts/analyze_results.sh
@@ -177,12 +177,12 @@ function print_log_comparison_insights
# For example, if count_occurrences_of_each_log_file identified that there
# were 10 instances of http logs across all of the `results.csv` files,
# ${OVERALL_LOG_CARDINALITY[http]} should equal 10. If
check_for_unequal_log_counts
- # independently found 10 instances where the http bro and kafka log counts
+ # independently found 10 instances where the http zeek and kafka log counts
# from the `results.csv` files were not equal, ${LOG_ISSUE_COUNT[http]}
# would also have 10 entries, causing us to warn the user of that insight.
for KEY in "${!LOG_ISSUE_COUNT[@]}"; do
if [[ "${LOG_ISSUE_COUNT[${KEY}]}" == "${OVERALL_LOG_CARDINALITY[${KEY}]}"
]]; then
- _echo WARN "None of the ${KEY} log counts were the same between bro and
kafka. This may indicate an issue specific to that log."
+ _echo WARN "None of the ${KEY} log counts were the same between zeek and
kafka. This may indicate an issue specific to that log."
fi
done
}
@@ -203,7 +203,7 @@ for file in $RESULTS_FILES; do
done
if [[ "${#LOGS_WITH_UNEQUAL_RESULTS[@]}" -gt 0 ]]; then
- _echo ERROR "UNEQUALITY FOUND IN BRO AND KAFKA LOG COUNTS"
+ _echo ERROR "UNEQUALITY FOUND IN ZEEK AND KAFKA LOG COUNTS"
echo ""
print_unequal_results
diff --git a/docker/scripts/docker_execute_build_bro_plugin.sh
b/docker/scripts/docker_execute_build_plugin.sh
similarity index 87%
rename from docker/scripts/docker_execute_build_bro_plugin.sh
rename to docker/scripts/docker_execute_build_plugin.sh
index 4c4d9ee..d1f8d7f 100755
--- a/docker/scripts/docker_execute_build_bro_plugin.sh
+++ b/docker/scripts/docker_execute_build_plugin.sh
@@ -24,20 +24,20 @@ set -E # errtrap
set -o pipefail
#
-# Executes the build_bro_plugin.sh script in the container
+# Executes the build_plugin.sh script in the container
#
function help {
echo " "
echo "usage: ${0}"
- echo " --container-name [OPTIONAL] The Docker container
name. Default: metron-bro-plugin-kafka_bro_1"
+ echo " --container-name [OPTIONAL] The Docker container
name. Default: metron-bro-plugin-kafka_zeek_1"
echo " --plugin-version [REQUIRED] The plugin version."
echo " -h/--help Usage information."
echo " "
echo " "
}
-CONTAINER_NAME="metron-bro-plugin-kafka_bro_1"
+CONTAINER_NAME="metron-bro-plugin-kafka_zeek_1"
PLUGIN_VERSION=
# handle command line options
@@ -88,14 +88,14 @@ if [[ -z "${PLUGIN_VERSION}" ]]; then
exit 1
fi
-echo "Running build_bro_plugin with "
+echo "Running build_plugin with "
echo "CONTAINER_NAME = $CONTAINER_NAME"
echo "==================================================="
-docker exec -w /root "${CONTAINER_NAME}" bash -c
"/root/built_in_scripts/build_bro_plugin.sh --plugin-version=${PLUGIN_VERSION}"
+docker exec -w /root "${CONTAINER_NAME}" bash -c
"/root/built_in_scripts/build_plugin.sh --plugin-version=${PLUGIN_VERSION}"
rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc};
fi
-echo "Built the bro plugin"
+echo "Built the plugin"
diff --git a/docker/scripts/docker_execute_configure_bro_plugin.sh
b/docker/scripts/docker_execute_configure_plugin.sh
similarity index 84%
copy from docker/scripts/docker_execute_configure_bro_plugin.sh
copy to docker/scripts/docker_execute_configure_plugin.sh
index ee62a58..0e853fe 100755
--- a/docker/scripts/docker_execute_configure_bro_plugin.sh
+++ b/docker/scripts/docker_execute_configure_plugin.sh
@@ -24,21 +24,21 @@ set -E # errtrap
set -o pipefail
#
-# Executes the configure_bro_plugin.sh in the docker container
+# Executes the configure_plugin.sh in the docker container
#
function help {
echo " "
echo "usage: ${0}"
- echo " --container-name [OPTIONAL] The Docker container
name. Default: metron-bro-plugin-kafka_bro_1"
- echo " --kafka-topic [OPTIONAL] The kafka topic to
create. Default: bro"
+ echo " --container-name [OPTIONAL] The Docker container
name. Default: metron-bro-plugin-kafka_zeek_1"
+ echo " --kafka-topic [OPTIONAL] The kafka topic to
create. Default: zeek"
echo " -h/--help Usage information."
echo " "
echo " "
}
-CONTAINER_NAME=metron-bro-plugin-kafka_bro_1
-KAFKA_TOPIC=bro
+CONTAINER_NAME=metron-bro-plugin-kafka_zeek_1
+KAFKA_TOPIC=zeek
# Handle command line options
for i in "$@"; do
@@ -80,15 +80,15 @@ for i in "$@"; do
esac
done
-echo "Running docker_execute_configure_bro_plugin.sh with "
+echo "Running docker_execute_configure_plugin.sh with "
echo "CONTAINER_NAME = ${CONTAINER_NAME}"
echo "KAFKA_TOPIC = ${KAFKA_TOPIC}"
echo "==================================================="
-docker exec -w /root "${CONTAINER_NAME}" bash -c
"/root/built_in_scripts/configure_bro_plugin.sh
--kafka-topic=\"${KAFKA_TOPIC}\""
+docker exec -w /root "${CONTAINER_NAME}" bash -c
"/root/built_in_scripts/configure_plugin.sh --kafka-topic=\"${KAFKA_TOPIC}\""
rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc};
fi
-echo "configured the bro kafka plugin"
+echo "configured the kafka plugin"
diff --git a/docker/scripts/docker_execute_configure_bro_plugin.sh
b/docker/scripts/docker_execute_configure_zeek_plugin.sh
similarity index 84%
rename from docker/scripts/docker_execute_configure_bro_plugin.sh
rename to docker/scripts/docker_execute_configure_zeek_plugin.sh
index ee62a58..0e853fe 100755
--- a/docker/scripts/docker_execute_configure_bro_plugin.sh
+++ b/docker/scripts/docker_execute_configure_zeek_plugin.sh
@@ -24,21 +24,21 @@ set -E # errtrap
set -o pipefail
#
-# Executes the configure_bro_plugin.sh in the docker container
+# Executes the configure_plugin.sh in the docker container
#
function help {
echo " "
echo "usage: ${0}"
- echo " --container-name [OPTIONAL] The Docker container
name. Default: metron-bro-plugin-kafka_bro_1"
- echo " --kafka-topic [OPTIONAL] The kafka topic to
create. Default: bro"
+ echo " --container-name [OPTIONAL] The Docker container
name. Default: metron-bro-plugin-kafka_zeek_1"
+ echo " --kafka-topic [OPTIONAL] The kafka topic to
create. Default: zeek"
echo " -h/--help Usage information."
echo " "
echo " "
}
-CONTAINER_NAME=metron-bro-plugin-kafka_bro_1
-KAFKA_TOPIC=bro
+CONTAINER_NAME=metron-bro-plugin-kafka_zeek_1
+KAFKA_TOPIC=zeek
# Handle command line options
for i in "$@"; do
@@ -80,15 +80,15 @@ for i in "$@"; do
esac
done
-echo "Running docker_execute_configure_bro_plugin.sh with "
+echo "Running docker_execute_configure_plugin.sh with "
echo "CONTAINER_NAME = ${CONTAINER_NAME}"
echo "KAFKA_TOPIC = ${KAFKA_TOPIC}"
echo "==================================================="
-docker exec -w /root "${CONTAINER_NAME}" bash -c
"/root/built_in_scripts/configure_bro_plugin.sh
--kafka-topic=\"${KAFKA_TOPIC}\""
+docker exec -w /root "${CONTAINER_NAME}" bash -c
"/root/built_in_scripts/configure_plugin.sh --kafka-topic=\"${KAFKA_TOPIC}\""
rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc};
fi
-echo "configured the bro kafka plugin"
+echo "configured the kafka plugin"
diff --git a/docker/scripts/docker_execute_create_topic_in_kafka.sh
b/docker/scripts/docker_execute_create_topic_in_kafka.sh
index 46435ef..470e160 100755
--- a/docker/scripts/docker_execute_create_topic_in_kafka.sh
+++ b/docker/scripts/docker_execute_create_topic_in_kafka.sh
@@ -27,13 +27,13 @@ function help {
echo " "
echo "usage: ${0}"
echo " --container-name [OPTIONAL] The Docker container
name. Default: metron-bro-plugin-kafka_kafka_1"
- echo " --kafka-topic [OPTIONAL] The kafka topic to
create. Default: bro"
+ echo " --kafka-topic [OPTIONAL] The kafka topic to
create. Default: zeek"
echo " -h/--help Usage information."
echo " "
}
CONTAINER_NAME="metron-bro-plugin-kafka_kafka_1"
-KAFKA_TOPIC=bro
+KAFKA_TOPIC=zeek
# handle command line options
for i in "$@"; do
diff --git a/docker/scripts/docker_execute_process_data_file.sh
b/docker/scripts/docker_execute_process_data_file.sh
index b227bd2..37d25c9 100755
--- a/docker/scripts/docker_execute_process_data_file.sh
+++ b/docker/scripts/docker_execute_process_data_file.sh
@@ -30,7 +30,7 @@ set -o pipefail
function help {
echo " "
echo "usage: ${0}"
- echo " --container-name [OPTIONAL] The Docker container
name. Default: metron-bro-plugin-kafka_bro_1"
+ echo " --container-name [OPTIONAL] The Docker container
name. Default: metron-bro-plugin-kafka_zeek_1"
echo " --pcap-file-name [REQUIRED] The name of the pcap
file"
echo " --output-directory-name [REQUIRED] The name of the output
directory"
echo " -h/--help Usage information."
@@ -38,7 +38,7 @@ function help {
echo " "
}
-CONTAINER_NAME=metron-bro-plugin-kafka_bro_1
+CONTAINER_NAME=metron-bro-plugin-kafka_zeek_1
PCAP_FILE_NAME=
OUTPUT_DIRECTORY_NAME=
@@ -101,7 +101,7 @@ echo "PCAP_FILE_NAME = ${PCAP_FILE_NAME}"
echo "OUTPUT_DIRECTORY_NAME = ${OUTPUT_DIRECTORY_NAME}"
echo "==================================================="
-echo "executing process_data_file.sh in the bro docker container"
+echo "executing process_data_file.sh in the zeek docker container"
echo " "
docker exec -w /root "${CONTAINER_NAME}" bash -c
"built_in_scripts/process_data_file.sh --pcap-file-name=${PCAP_FILE_NAME}
--output-directory-name=${OUTPUT_DIRECTORY_NAME}"
diff --git a/docker/scripts/docker_execute_shell.sh
b/docker/scripts/docker_execute_shell.sh
index c280e8c..d9daf79 100755
--- a/docker/scripts/docker_execute_shell.sh
+++ b/docker/scripts/docker_execute_shell.sh
@@ -30,13 +30,13 @@ set -o pipefail
function help {
echo " "
echo "usage: ${0}"
- echo " --container-name [OPTIONAL] The Docker container
name. Default: metron-bro-plugin-kafka_bro_1"
+ echo " --container-name [OPTIONAL] The Docker container
name. Default: metron-bro-plugin-kafka_zeek_1"
echo " -h/--help Usage information."
echo " "
echo " "
}
-CONTAINER_NAME=metron-bro-plugin-kafka_bro_1
+CONTAINER_NAME=metron-bro-plugin-kafka_zeek_1
# handle command line options
for i in "$@"; do
diff --git a/docker/scripts/docker_run_consume_kafka.sh
b/docker/scripts/docker_run_consume_kafka.sh
index a45b218..e8ac98e 100755
--- a/docker/scripts/docker_run_consume_kafka.sh
+++ b/docker/scripts/docker_run_consume_kafka.sh
@@ -33,14 +33,14 @@ function help {
echo "usage: ${0}"
echo " --network-name [OPTIONAL] The Docker network
name. Default: metron-bro-plugin-kafka_default"
echo " --offset [OPTIONAL] The kafka offset to
read from. Default: -1"
- echo " --kafka-topic [OPTIONAL] The kafka topic to
consume from. Default: bro"
+ echo " --kafka-topic [OPTIONAL] The kafka topic to
consume from. Default: zeek"
echo " -h/--help Usage information."
echo " "
}
NETWORK_NAME=metron-bro-plugin-kafka_default
OFFSET=-1
-KAFKA_TOPIC=bro
+KAFKA_TOPIC=zeek
# handle command line options
for i in "$@"; do
diff --git a/docker/scripts/docker_run_get_offset_kafka.sh
b/docker/scripts/docker_run_get_offset_kafka.sh
index 5a2c2ba..6f53926 100755
--- a/docker/scripts/docker_run_get_offset_kafka.sh
+++ b/docker/scripts/docker_run_get_offset_kafka.sh
@@ -32,13 +32,13 @@ function help {
echo " "
echo "usage: ${0}"
echo " --network-name [OPTIONAL] The Docker network
name. Default: metron-bro-plugin-kafka_default"
- echo " --kafka-topic [OPTIONAL] The kafka topic to pull
the offset from. Default: bro"
+ echo " --kafka-topic [OPTIONAL] The kafka topic to pull
the offset from. Default: zeek"
echo " -h/--help Usage information."
echo " "
}
NETWORK_NAME=metron-bro-plugin-kafka_default
-KAFKA_TOPIC=bro
+KAFKA_TOPIC=zeek
# handle command line options
for i in "$@"; do
diff --git a/docker/scripts/download_sample_pcaps.sh
b/docker/scripts/download_sample_pcaps.sh
index 408dcac..addc93f 100755
--- a/docker/scripts/download_sample_pcaps.sh
+++ b/docker/scripts/download_sample_pcaps.sh
@@ -87,7 +87,7 @@ for folder in nitroba example-traffic ssh ftp radius rfb; do
done
if [[ ! -f "${DATA_PATH}"/example-traffic/exercise-traffic.pcap ]]; then
- wget
https://github.com/bro/try-bro/raw/master/manager/static/pcaps/exercise_traffic.pcap
-O "${DATA_PATH}"/example-traffic/exercise-traffic.pcap
+ wget
https://github.com/zeek/try-zeek/raw/master/manager/static/pcaps/exercise_traffic.pcap
-O "${DATA_PATH}"/example-traffic/exercise-traffic.pcap
fi
if [[ ! -f "${DATA_PATH}"/nitroba/nitroba.pcap ]]; then
@@ -95,7 +95,7 @@ if [[ ! -f "${DATA_PATH}"/nitroba/nitroba.pcap ]]; then
fi
if [[ ! -f "${DATA_PATH}"/ssh/ssh.pcap ]]; then
- wget https://github.com/bro/try-bro/raw/master/manager/static/pcaps/ssh.pcap
-O "${DATA_PATH}"/ssh/ssh.pcap
+ wget
https://github.com/zeek/try-zeek/raw/master/manager/static/pcaps/ssh.pcap -O
"${DATA_PATH}"/ssh/ssh.pcap
fi
if [[ ! -f "${DATA_PATH}"/ftp/ftp.pcap ]]; then
diff --git a/docker/scripts/split_kafka_output_by_log.sh
b/docker/scripts/split_kafka_output_by_log.sh
index 61e53e4..04139a9 100755
--- a/docker/scripts/split_kafka_output_by_log.sh
+++ b/docker/scripts/split_kafka_output_by_log.sh
@@ -24,8 +24,8 @@ set -E # errtrap
set -o pipefail
#
-# For a given directory, finds all the bro log output, and splits the kafka
output file
-# by bro log, such that there is a bro log -> bro log kafka log
+# For a given directory, finds all the zeek log output, and splits the kafka
+# output file by zeek log, such that there is a zeek log -> zeek log kafka log
#
function help {
@@ -89,7 +89,7 @@ cd "${LOG_DIRECTORY}" || exit 1
# and extract the start
# then we want to grep that name > name.kafka.log from the KAFKA_OUTPUT_FILE
RESULTS_FILE="${LOG_DIRECTORY}/results.csv"
-echo "LOG,BRO_COUNT,KAFKA_COUNT" >> "${RESULTS_FILE}"
+echo "LOG,ZEEK_COUNT,KAFKA_COUNT" >> "${RESULTS_FILE}"
for log in "${LOG_DIRECTORY}"/*.log
do
BASE_LOG_FILE_NAME=$(basename "$log" .log)
@@ -98,9 +98,9 @@ do
grep {\""${BASE_LOG_FILE_NAME}"\": "${LOG_DIRECTORY}"/kafka-output.log >
"${LOG_DIRECTORY}"/"${BASE_LOG_FILE_NAME}".kafka.log
KAKFA_COUNT=$(cat "${LOG_DIRECTORY}/${BASE_LOG_FILE_NAME}.kafka.log" |
wc -l)
- BRO_COUNT=$(grep -v "^#" "${log}" | wc -l)
+ ZEEK_COUNT=$(grep -v "^#" "${log}" | wc -l)
- echo "${BASE_LOG_FILE_NAME},${BRO_COUNT},${KAKFA_COUNT}" >>
"${RESULTS_FILE}"
+ echo "${BASE_LOG_FILE_NAME},${ZEEK_COUNT},${KAKFA_COUNT}" >>
"${RESULTS_FILE}"
fi
fi
done
diff --git a/scripts/Apache/Kafka/__load__.bro
b/scripts/Apache/Kafka/__load__.zeek
similarity index 97%
rename from scripts/Apache/Kafka/__load__.bro
rename to scripts/Apache/Kafka/__load__.zeek
index f5a85d6..95b4c84 100644
--- a/scripts/Apache/Kafka/__load__.bro
+++ b/scripts/Apache/Kafka/__load__.zeek
@@ -20,4 +20,4 @@
# loaded automatically at that point.
#
-@load ./logs-to-kafka.bro
+@load ./logs-to-kafka.zeek
diff --git a/scripts/Apache/Kafka/logs-to-kafka.bro
b/scripts/Apache/Kafka/logs-to-kafka.zeek
similarity index 98%
rename from scripts/Apache/Kafka/logs-to-kafka.bro
rename to scripts/Apache/Kafka/logs-to-kafka.zeek
index 5852505..6831ed2 100644
--- a/scripts/Apache/Kafka/logs-to-kafka.bro
+++ b/scripts/Apache/Kafka/logs-to-kafka.zeek
@@ -36,7 +36,7 @@ function send_to_kafka(id: Log::ID): bool
return T;
}
-event bro_init() &priority=-10
+event zeek_init() &priority=-10
{
for (stream_id in Log::active_streams)
{
@@ -55,4 +55,4 @@ event bro_init() &priority=-10
event kafka_topic_resolved_event(topic: string) {
print(fmt("Kafka topic set to %s",topic));
-}
\ No newline at end of file
+}
diff --git a/scripts/__load__.bro b/scripts/__load__.zeek
similarity index 64%
copy from scripts/__load__.bro
copy to scripts/__load__.zeek
index e3db306..41f2499 100644
--- a/scripts/__load__.bro
+++ b/scripts/__load__.zeek
@@ -16,12 +16,13 @@
#
#
-# This is loaded unconditionally at Bro startup. Include scripts here that
should
-# always be loaded.
+# This is loaded automatically at Zeek startup once the plugin gets activated
+# and its BiF elements have become available. Include code here that should
+# always execute unconditionally at that time.
#
-# Normally, that will be only code that initializes built-in elements. Load
-# your standard scripts in
-# scripts/<plugin-namespace>/<plugin-name>/__load__.bro instead.
+# Note that often you may want your plugin's accompanying scripts not here, but
+# in scripts/<plugin-namespace>/<plugin-name>/__load__.zeek. That's processed
+# only on explicit `@load <plugin-namespace>/<plugin-name>`.
#
-@load ./init.bro
+@load ./init.zeek
diff --git a/scripts/init.bro b/scripts/init.zeek
similarity index 91%
rename from scripts/init.bro
rename to scripts/init.zeek
index 7e2c56c..6f5a7ae 100644
--- a/scripts/init.bro
+++ b/scripts/init.zeek
@@ -24,12 +24,12 @@ export {
## Example: redef Kafka::send_all_active_logs = T;
const send_all_active_logs: bool = F &redef;
- ## Specify which :bro:type:`Log::ID` to send to kafka.
+ ## Specify which :zeek:type:`Log::ID` to send to kafka.
##
## Example: redef Kafka::logs_to_send = set(Conn::Log, DNS::LOG);
const logs_to_send: set[Log::ID] &redef;
- ## Specify which :bro:type:`Log::ID` to exclude from being sent to
kafka.
+ ## Specify which :zeek:type:`Log::ID` to exclude from being sent to
kafka.
##
## Example: redef Kafka::logs_to_exclude = set(SSH::LOG);
const logs_to_exclude: set[Log::ID] &redef;
@@ -40,7 +40,7 @@ export {
const json_timestamps: JSON::TimestampFormat = JSON::TS_EPOCH &redef;
## Destination kafka topic name
- const topic_name: string = "bro" &redef;
+ const topic_name: string = "zeek" &redef;
## Maximum wait on shutdown in milliseconds
const max_wait_on_shutdown: count = 3000 &redef;
diff --git a/src/KafkaWriter.cc b/src/KafkaWriter.cc
index d2287bf..62e4ac3 100644
--- a/src/KafkaWriter.cc
+++ b/src/KafkaWriter.cc
@@ -29,7 +29,7 @@ KafkaWriter::KafkaWriter(WriterFrontend* frontend):
topic(NULL)
{
/**
- * We need thread-local copies of all user-defined settings coming from bro
+ * We need thread-local copies of all user-defined settings coming from zeek
* scripting land. accessing these is not thread-safe and 'DoInit' is
* potentially accessed from multiple threads.
*/
@@ -94,8 +94,8 @@ bool KafkaWriter::DoInit(const WriterInfo& info, int
num_fields, const threading
// Timeformat object, default to TS_EPOCH
threading::formatter::JSON::TimeFormat tf =
threading::formatter::JSON::TS_EPOCH;
- // Allow overriding of the kafka topic via the Bro script constant
'topic_name'
- // which can be applied when adding a new Bro log filter.
+ // Allow overriding of the kafka topic via the Zeek script constant
+ // 'topic_name' which can be applied when adding a new Zeek log filter.
topic_name_override = GetConfigValue(info, "topic_name");
if(!topic_name_override.empty()) {
@@ -114,8 +114,8 @@ bool KafkaWriter::DoInit(const WriterInfo& info, int
num_fields, const threading
/**
* Format the timestamps
* NOTE: This string comparision implementation is currently the necessary
- * way to do it, as there isn't a way to pass the Bro enum into C++ enum.
- * This makes the user interface consistent with the existing Bro Logging
+ * way to do it, as there isn't a way to pass the Zeek enum into C++ enum.
+ * This makes the user interface consistent with the existing Zeek Logging
* configuration for the ASCII log output.
*/
if ( strcmp(json_timestamps.c_str(), "JSON::TS_EPOCH") == 0 ) {
@@ -230,10 +230,10 @@ bool KafkaWriter::DoFinish(double network_time)
delete topic;
delete producer;
+ delete topic_conf;
}
delete formatter;
delete conf;
- delete topic_conf;
return success;
}
@@ -268,7 +268,7 @@ bool KafkaWriter::DoWrite(int num_fields, const
threading::Field* const* fields,
}
/**
- * Writer-specific method implementing a change of fthe buffering
+ * Writer-specific method implementing a change of the buffering
* state. If buffering is disabled, the writer should attempt to
* write out information as quickly as possible even if doing so may
* have a performance impact. If enabled (which is the default), it
diff --git a/src/KafkaWriter.h b/src/KafkaWriter.h
index 0ef0fb1..e9193f8 100644
--- a/src/KafkaWriter.h
+++ b/src/KafkaWriter.h
@@ -15,12 +15,13 @@
* limitations under the License.
*/
-#ifndef BRO_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
-#define BRO_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
+#ifndef ZEEK_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
+#define ZEEK_PLUGIN_BRO_KAFKA_KAFKAWRITER_H
#include <librdkafka/rdkafkacpp.h>
#include <string>
#include <Type.h>
+#include <Desc.h>
#include <logging/WriterBackend.h>
#include <threading/formatters/JSON.h>
#include <threading/Formatter.h>
diff --git a/src/Plugin.h b/src/Plugin.h
index e99e91b..e622c54 100644
--- a/src/Plugin.h
+++ b/src/Plugin.h
@@ -15,8 +15,8 @@
* limitations under the License.
*/
-#ifndef BRO_PLUGIN_BRO_KAFKA
-#define BRO_PLUGIN_BRO_KAFKA
+#ifndef ZEEK_PLUGIN_BRO_KAFKA
+#define ZEEK_PLUGIN_BRO_KAFKA
#include "KafkaWriter.h"
#include <plugin/Plugin.h>
diff --git a/src/TaggedJSON.h b/src/TaggedJSON.h
index 08a50df..51b1bf3 100644
--- a/src/TaggedJSON.h
+++ b/src/TaggedJSON.h
@@ -15,10 +15,11 @@
* limitations under the License.
*/
-#ifndef BRO_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
-#define BRO_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
+#ifndef ZEEK_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
+#define ZEEK_PLUGIN_BRO_KAFKA_TAGGEDJSON_H
#include <string>
+#include <Desc.h>
#include <threading/Formatter.h>
#include <threading/formatters/JSON.h>
diff --git a/tests/.gitignore b/tests/.gitignore
new file mode 100644
index 0000000..fc422ef
--- /dev/null
+++ b/tests/.gitignore
@@ -0,0 +1,2 @@
+.btest.failed.dat
+.tmp
diff --git a/tests/Baseline/kafka.resolved-topic-default/output
b/tests/Baseline/kafka.resolved-topic-default/output
index 1cfb642..73b9458 100644
--- a/tests/Baseline/kafka.resolved-topic-default/output
+++ b/tests/Baseline/kafka.resolved-topic-default/output
@@ -1 +1 @@
-Kafka topic set to bro
+Kafka topic set to zeek
diff --git a/scripts/__load__.bro b/tests/Scripts/diff-remove-timestamps
old mode 100644
new mode 100755
similarity index 60%
rename from scripts/__load__.bro
rename to tests/Scripts/diff-remove-timestamps
index e3db306..b8fd2e4
--- a/scripts/__load__.bro
+++ b/tests/Scripts/diff-remove-timestamps
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -14,14 +15,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
-#
-# This is loaded unconditionally at Bro startup. Include scripts here that
should
-# always be loaded.
-#
-# Normally, that will be only code that initializes built-in elements. Load
-# your standard scripts in
-# scripts/<plugin-namespace>/<plugin-name>/__load__.bro instead.
+# The upstream of this file is at
+#
https://github.com/zeek/zeek-aux/blob/master/plugin-support/skeleton/tests/Scripts/diff-remove-timestamps
#
+# Replace anything which looks like timestamps with XXXs (including the
#start/end markers in logs).
+
+# Get us "modern" regexps with sed.
+if [ `uname` == "Linux" ]; then
+ sed="sed -r"
+else
+ sed="sed -E"
+fi
-@load ./init.bro
+$sed 's/(0\.000000)|([0-9]{9,10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \
+$sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'
diff --git a/tests/Scripts/get-bro-env b/tests/Scripts/get-bro-env
deleted file mode 100755
index 8aa0ea7..0000000
--- a/tests/Scripts/get-bro-env
+++ /dev/null
@@ -1,36 +0,0 @@
-#! /bin/sh
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# BTest helper for getting values for Bro-related environment variables.
-
-base=`dirname $0`
-bro=`cat ${base}/../../build/CMakeCache.txt | grep BRO_DIST | cut -d = -f 2`
-
-if [ "$1" = "brobase" ]; then
- echo ${bro}
-elif [ "$1" = "bropath" ]; then
- ${bro}/build/bro-path-dev
-elif [ "$1" = "bro_plugin_path" ]; then
- ( cd ${base}/../.. && pwd )
-elif [ "$1" = "bro_seed_file" ]; then
- echo ${bro}/testing/btest/random.seed
-elif [ "$1" = "path" ]; then
- echo ${bro}/build/src:${bro}/aux/btest:${base}/:${bro}/aux/bro-cut:$PATH
-else
- echo "usage: `basename $0` <var>" >&2
- exit 1
-fi
diff --git a/tests/Scripts/get-zeek-env b/tests/Scripts/get-zeek-env
new file mode 100755
index 0000000..8e3bb31
--- /dev/null
+++ b/tests/Scripts/get-zeek-env
@@ -0,0 +1,51 @@
+#! /bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# BTest helper for getting values for Zeek-related environment variables.
+
+base=`dirname $0`
+zeek_dist=`cat ${base}/../../build/CMakeCache.txt | grep ZEEK_DIST | cut -d =
-f 2`
+
+if [ -n "${zeek_dist}" ]; then
+ if [ "$1" = "zeekpath" ]; then
+ ${zeek_dist}/build/zeek-path-dev
+ elif [ "$1" = "zeek_plugin_path" ]; then
+ ( cd ${base}/../.. && pwd )
+ elif [ "$1" = "path" ]; then
+ echo
${zeek_dist}/build/src:${zeek_dist}/aux/btest:${base}/:${zeek_dist}/aux/zeek-cut:$PATH
+ else
+ echo "usage: `basename $0` <var>" >&2
+ exit 1
+ fi
+else
+ # Use Zeek installation for testing. In this case zeek-config must be in
PATH.
+ if ! which zeek-config >/dev/null; then
+ echo "zeek-config not found" >&2
+ exit 1
+ fi
+
+ if [ "$1" = "zeekpath" ]; then
+ zeek-config --zeekpath
+ elif [ "$1" = "zeek_plugin_path" ]; then
+ ( cd ${base}/../.. && pwd )
+ elif [ "$1" = "path" ]; then
+ echo ${PATH}
+ else
+ echo "usage: `basename $0` <var>" >&2
+ exit 1
+ fi
+fi
diff --git a/tests/btest.cfg b/tests/btest.cfg
index e42fefd..0071d26 100644
--- a/tests/btest.cfg
+++ b/tests/btest.cfg
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+# The upstream of this file is at
+#
https://github.com/zeek/zeek-aux/blob/master/plugin-support/skeleton/tests/btest.cfg
[btest]
TestDirs = kafka
@@ -23,14 +25,12 @@ IgnoreDirs = .svn CVS .tmp
IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store
[environment]
-BROBASE=`%(testbase)s/Scripts/get-bro-env brobase`
-BROPATH=`%(testbase)s/Scripts/get-bro-env bropath`
-BRO_PLUGIN_PATH=`%(testbase)s/Scripts/get-bro-env bro_plugin_path`
-BRO_SEED_FILE=`%(testbase)s/Scripts/get-bro-env bro_seed_file`
-PATH=`%(testbase)s/Scripts/get-bro-env path`
+ZEEKPATH=`%(testbase)s/Scripts/get-zeek-env zeekpath`
+ZEEK_PLUGIN_PATH=`%(testbase)s/Scripts/get-zeek-env zeek_plugin_path`
+ZEEK_SEED_FILE=%(testbase)s/random.seed
+PATH=`%(testbase)s/Scripts/get-zeek-env path`
TZ=UTC
LC_ALL=C
TRACES=%(testbase)s/Traces
TMPDIR=%(testbase)s/.tmp
-BRO_TRACES=`%(testbase)s/Scripts/get-bro-env brobase`/testing/btest/Traces
-TEST_DIFF_CANONIFIER=`%(testbase)s/Scripts/get-bro-env
brobase`/testing/scripts/diff-canonifier
+TEST_DIFF_CANONIFIER=%(testbase)s/Scripts/diff-remove-timestamps
diff --git a/tests/kafka/l2s-l2e-no-overlap.bro
b/tests/kafka/l2s-l2e-no-overlap.zeek
similarity index 94%
rename from tests/kafka/l2s-l2e-no-overlap.bro
rename to tests/kafka/l2s-l2e-no-overlap.zeek
index a0ede70..8b3b749 100644
--- a/tests/kafka/l2s-l2e-no-overlap.bro
+++ b/tests/kafka/l2s-l2e-no-overlap.zeek
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# @TEST-EXEC: bro ../../../scripts/Apache/Kafka/ %INPUT > output
+# @TEST-EXEC: zeek ../../../scripts/Apache/Kafka/ %INPUT > output
# @TEST-EXEC: btest-diff output
module Kafka;
diff --git a/tests/kafka/l2s-set-l2e-set.bro b/tests/kafka/l2s-set-l2e-set.zeek
similarity index 94%
rename from tests/kafka/l2s-set-l2e-set.bro
rename to tests/kafka/l2s-set-l2e-set.zeek
index a13c68d..bad509f 100644
--- a/tests/kafka/l2s-set-l2e-set.bro
+++ b/tests/kafka/l2s-set-l2e-set.zeek
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# @TEST-EXEC: bro ../../../scripts/Apache/Kafka/ %INPUT > output
+# @TEST-EXEC: zeek ../../../scripts/Apache/Kafka/ %INPUT > output
# @TEST-EXEC: btest-diff output
module Kafka;
diff --git a/tests/kafka/l2s-set-l2e-unset.bro
b/tests/kafka/l2s-set-l2e-unset.zeek
similarity index 93%
rename from tests/kafka/l2s-set-l2e-unset.bro
rename to tests/kafka/l2s-set-l2e-unset.zeek
index 439f578..4b0ea13 100644
--- a/tests/kafka/l2s-set-l2e-unset.bro
+++ b/tests/kafka/l2s-set-l2e-unset.zeek
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# @TEST-EXEC: bro ../../../scripts/Apache/Kafka/ %INPUT > output
+# @TEST-EXEC: zeek ../../../scripts/Apache/Kafka/ %INPUT > output
# @TEST-EXEC: btest-diff output
module Kafka;
diff --git a/tests/kafka/l2s-unset-l2e-set.bro
b/tests/kafka/l2s-unset-l2e-set.zeek
similarity index 93%
rename from tests/kafka/l2s-unset-l2e-set.bro
rename to tests/kafka/l2s-unset-l2e-set.zeek
index 3898e3a..1379f1a 100644
--- a/tests/kafka/l2s-unset-l2e-set.bro
+++ b/tests/kafka/l2s-unset-l2e-set.zeek
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# @TEST-EXEC: bro ../../../scripts/Apache/Kafka/ %INPUT > output
+# @TEST-EXEC: zeek ../../../scripts/Apache/Kafka/ %INPUT > output
# @TEST-EXEC: btest-diff output
module Kafka;
diff --git a/tests/kafka/l2s-unset-l2e-unset.bro
b/tests/kafka/l2s-unset-l2e-unset.zeek
similarity index 93%
rename from tests/kafka/l2s-unset-l2e-unset.bro
rename to tests/kafka/l2s-unset-l2e-unset.zeek
index 7fd6b9f..1288434 100644
--- a/tests/kafka/l2s-unset-l2e-unset.bro
+++ b/tests/kafka/l2s-unset-l2e-unset.zeek
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# @TEST-EXEC: bro ../../../scripts/Apache/Kafka/ %INPUT > output
+# @TEST-EXEC: zeek ../../../scripts/Apache/Kafka/ %INPUT > output
# @TEST-EXEC: btest-diff output
module Kafka;
diff --git a/tests/kafka/resolved-topic-config.bro
b/tests/kafka/resolved-topic-config.zeek
similarity index 89%
rename from tests/kafka/resolved-topic-config.bro
rename to tests/kafka/resolved-topic-config.zeek
index 56fa093..5c5f5d4 100644
--- a/tests/kafka/resolved-topic-config.bro
+++ b/tests/kafka/resolved-topic-config.zeek
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# @TEST-EXEC: bro -r ../../../tests/pcaps/exercise-traffic.pcap
../../../scripts/Apache/Kafka/ %INPUT > output
+# @TEST-EXEC: zeek -r ../../../tests/pcaps/exercise-traffic.pcap
../../../scripts/Apache/Kafka/ %INPUT > output
# @TEST-EXEC: btest-diff output
module Kafka;
diff --git a/tests/kafka/resolved-topic-default.bro
b/tests/kafka/resolved-topic-default.zeek
similarity index 89%
rename from tests/kafka/resolved-topic-default.bro
rename to tests/kafka/resolved-topic-default.zeek
index ea9d217..9ce5d9e 100644
--- a/tests/kafka/resolved-topic-default.bro
+++ b/tests/kafka/resolved-topic-default.zeek
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# @TEST-EXEC: bro -r ../../../tests/pcaps/exercise-traffic.pcap
../../../scripts/Apache/Kafka/ %INPUT > output
+# @TEST-EXEC: zeek -r ../../../tests/pcaps/exercise-traffic.pcap
../../../scripts/Apache/Kafka/ %INPUT > output
# @TEST-EXEC: btest-diff output
module Kafka;
diff --git a/tests/kafka/resolved-topic-override-and-config.bro
b/tests/kafka/resolved-topic-override-and-config.zeek
similarity index 89%
rename from tests/kafka/resolved-topic-override-and-config.bro
rename to tests/kafka/resolved-topic-override-and-config.zeek
index d75ce61..37d1258 100644
--- a/tests/kafka/resolved-topic-override-and-config.bro
+++ b/tests/kafka/resolved-topic-override-and-config.zeek
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# @TEST-EXEC: bro -r ../../../tests/pcaps/exercise-traffic.pcap
../../../scripts/Apache/Kafka/ %INPUT > output
+# @TEST-EXEC: zeek -r ../../../tests/pcaps/exercise-traffic.pcap
../../../scripts/Apache/Kafka/ %INPUT > output
# @TEST-EXEC: btest-diff output
module Kafka;
@@ -25,7 +25,7 @@ redef Kafka::logs_to_send = set(Conn::LOG);
redef Kafka::topic_name = "const-variable-topic";
redef Kafka::mock = T;
-event bro_init() &priority=-10
+event zeek_init() &priority=-10
{
local xxx_filter: Log::Filter = [
$name = "kafka-xxx",
diff --git a/tests/kafka/resolved-topic-override-only.bro
b/tests/kafka/resolved-topic-override-only.zeek
similarity index 88%
rename from tests/kafka/resolved-topic-override-only.bro
rename to tests/kafka/resolved-topic-override-only.zeek
index 35cf606..3313b5b 100644
--- a/tests/kafka/resolved-topic-override-only.bro
+++ b/tests/kafka/resolved-topic-override-only.zeek
@@ -15,14 +15,14 @@
# limitations under the License.
#
-# @TEST-EXEC: bro -r ../../../tests/pcaps/exercise-traffic.pcap
../../../scripts/Apache/Kafka/ %INPUT > output
+# @TEST-EXEC: zeek -r ../../../tests/pcaps/exercise-traffic.pcap
../../../scripts/Apache/Kafka/ %INPUT > output
# @TEST-EXEC: btest-diff output
module Kafka;
redef Kafka::mock = T;
-event bro_init() &priority=-10
+event zeek_init() &priority=-10
{
local xxx_filter: Log::Filter = [
$name = "kafka-xxx",
diff --git a/tests/kafka/send-all-active-logs-l2e-set.bro
b/tests/kafka/send-all-active-logs-l2e-set.zeek
similarity index 94%
rename from tests/kafka/send-all-active-logs-l2e-set.bro
rename to tests/kafka/send-all-active-logs-l2e-set.zeek
index 9019702..10fee14 100644
--- a/tests/kafka/send-all-active-logs-l2e-set.bro
+++ b/tests/kafka/send-all-active-logs-l2e-set.zeek
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# @TEST-EXEC: bro ../../../scripts/Apache/Kafka/ %INPUT > output
+# @TEST-EXEC: zeek ../../../scripts/Apache/Kafka/ %INPUT > output
# @TEST-EXEC: btest-diff output
module Kafka;
diff --git a/tests/kafka/send-all-active-logs-l2e-unset.bro
b/tests/kafka/send-all-active-logs-l2e-unset.zeek
similarity index 94%
rename from tests/kafka/send-all-active-logs-l2e-unset.bro
rename to tests/kafka/send-all-active-logs-l2e-unset.zeek
index afef422..321a850 100644
--- a/tests/kafka/send-all-active-logs-l2e-unset.bro
+++ b/tests/kafka/send-all-active-logs-l2e-unset.zeek
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# @TEST-EXEC: bro ../../../scripts/Apache/Kafka/ %INPUT > output
+# @TEST-EXEC: zeek ../../../scripts/Apache/Kafka/ %INPUT > output
# @TEST-EXEC: btest-diff output
module Kafka;
diff --git a/tests/kafka/send-all-active-logs-l2s-set-l2e-set.bro
b/tests/kafka/send-all-active-logs-l2s-set-l2e-set.zeek
similarity index 94%
rename from tests/kafka/send-all-active-logs-l2s-set-l2e-set.bro
rename to tests/kafka/send-all-active-logs-l2s-set-l2e-set.zeek
index 6d223e2..69763a3 100644
--- a/tests/kafka/send-all-active-logs-l2s-set-l2e-set.bro
+++ b/tests/kafka/send-all-active-logs-l2s-set-l2e-set.zeek
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# @TEST-EXEC: bro ../../../scripts/Apache/Kafka/ %INPUT > output
+# @TEST-EXEC: zeek ../../../scripts/Apache/Kafka/ %INPUT > output
# @TEST-EXEC: btest-diff output
module Kafka;
diff --git a/tests/kafka/send-all-active-logs-l2s-set-l2e-unset.bro
b/tests/kafka/send-all-active-logs-l2s-set-l2e-unset.zeek
similarity index 94%
rename from tests/kafka/send-all-active-logs-l2s-set-l2e-unset.bro
rename to tests/kafka/send-all-active-logs-l2s-set-l2e-unset.zeek
index cc7788d..8d4dc0b 100644
--- a/tests/kafka/send-all-active-logs-l2s-set-l2e-unset.bro
+++ b/tests/kafka/send-all-active-logs-l2s-set-l2e-unset.zeek
@@ -15,7 +15,7 @@
# limitations under the License.
#
-# @TEST-EXEC: bro ../../../scripts/Apache/Kafka/ %INPUT > output
+# @TEST-EXEC: zeek ../../../scripts/Apache/Kafka/ %INPUT > output
# @TEST-EXEC: btest-diff output
module Kafka;
diff --git a/tests/kafka/show-plugin.bro b/tests/kafka/show-plugin.zeek
similarity index 92%
rename from tests/kafka/show-plugin.bro
rename to tests/kafka/show-plugin.zeek
index ba065cc..6161a36 100644
--- a/tests/kafka/show-plugin.bro
+++ b/tests/kafka/show-plugin.zeek
@@ -15,5 +15,5 @@
# limitations under the License.
#
-# @TEST-EXEC: bro -NN Apache::Kafka | sed 's/, version.*)/)/' > output
+# @TEST-EXEC: zeek -NN Apache::Kafka | sed 's/, version.*)/)/' > output
# @TEST-EXEC: btest-diff output
diff --git a/tests/random.seed b/tests/random.seed
new file mode 100644
index 0000000..6956a2c
--- /dev/null
+++ b/tests/random.seed
@@ -0,0 +1,21 @@
+2983378351
+1299727368
+0
+310447
+0
+1409073626
+3975311262
+34130240
+1450515018
+1466150520
+1342286698
+1193956778
+2188527278
+3361989254
+3912865238
+3596260151
+517973768
+1462428821
+0
+2278350848
+32767
diff --git a/zkg.meta b/zkg.meta
new file mode 100644
index 0000000..679daa3
--- /dev/null
+++ b/zkg.meta
@@ -0,0 +1,15 @@
+[package]
+description = A Zeek log writer plugin that sends logging output to Kafka.
+tags = log writer, zeek plugin, kafka
+script_dir = build/scripts/Apache/Kafka
+build_command = ./configure --zeek-dist=%(zeek_dist)s
--with-librdkafka=%(LIBRDKAFKA_ROOT)s && make
+test_command = ( cd tests && btest -d )
+plugin_dir = build
+version = 0.3
+depends =
+ zeek >=3.0.0
+ zkg >=2.0
+external_depends =
+ librdkafka ~1.4.2-RC1
+user_vars =
+ LIBRDKAFKA_ROOT [/usr/local/lib] "Path to librdkafka installation tree"
