[metron-bro-plugin-kafka] branch master updated: METRON-2360 BRO-PLUGIN: does not build with 3.2.1 (ottobackwards) closes apache/metron-bro-plugin-kafka#48
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new 92b85e5 METRON-2360 BRO-PLUGIN: does not build with 3.2.1
(ottobackwards) closes apache/metron-bro-plugin-kafka#48
92b85e5 is described below
commit 92b85e5e00cc2fd9023ea7f53466db6592eb6634
Author: ottobackwards
AuthorDate: Tue Sep 22 12:46:41 2020 -0400
METRON-2360 BRO-PLUGIN: does not build with 3.2.1 (ottobackwards) closes
apache/metron-bro-plugin-kafka#48
---
docker/containers/zeek/Dockerfile | 2 +-
docker/docker-compose.yml | 2 +-
src/KafkaWriter.cc| 430 ++
src/KafkaWriter.h | 19 +-
src/TaggedJSON.cc | 9 +-
src/TaggedJSON.h | 23 +-
6 files changed, 238 insertions(+), 247 deletions(-)
diff --git a/docker/containers/zeek/Dockerfile
b/docker/containers/zeek/Dockerfile
index dba31d7..d8eda31 100644
--- a/docker/containers/zeek/Dockerfile
+++ b/docker/containers/zeek/Dockerfile
@@ -57,7 +57,7 @@ ENV PATH="${PATH}:/usr/bin"
# install pip3 and zkg
WORKDIR /root
COPY requirements.txt requirements.txt
-RUN dnf -y install python3-pip && \
+RUN dnf -y install python3-pip diffutils && \
dnf clean all && \
python3 -m pip install --upgrade pip && \
python3 -m pip install -r requirements.txt && \
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 0579887..4f8ba10 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -29,7 +29,7 @@ services:
build:
context: containers/zeek
args:
-ZEEK_VERSION: "3.1.3"
+ZEEK_VERSION: "3.2.1"
LIBRDKAFKA_VERSION: "1.4.2"
image: metron-bro-plugin-kafka_zeek:latest
depends_on:
diff --git a/src/KafkaWriter.cc b/src/KafkaWriter.cc
index deeea95..1e19b3b 100644
--- a/src/KafkaWriter.cc
+++ b/src/KafkaWriter.cc
@@ -22,12 +22,8 @@ using namespace logging;
using namespace writer;
// The Constructor is called once for each log filter that uses this log
writer.
-KafkaWriter::KafkaWriter(WriterFrontend* frontend):
-WriterBackend(frontend),
-formatter(NULL),
-producer(NULL),
-topic(NULL)
-{
+KafkaWriter::KafkaWriter(WriterFrontend *frontend)
+: WriterBackend(frontend), formatter(NULL), producer(NULL), topic(NULL) {
/**
* We need thread-local copies of all user-defined settings coming from zeek
* scripting land. accessing these is not thread-safe and 'DoInit' is
@@ -41,177 +37,170 @@ KafkaWriter::KafkaWriter(WriterFrontend* frontend):
// json_timestamps
ODesc tsfmt;
BifConst::Kafka::json_timestamps->Describe();
- json_timestamps.assign(
- (const char*) tsfmt.Bytes(),
- tsfmt.Len()
-);
+ json_timestamps.assign((const char *)tsfmt.Bytes(), tsfmt.Len());
// topic name - thread local copy
- topic_name.assign(
-(const char*)BifConst::Kafka::topic_name->Bytes(),
-BifConst::Kafka::topic_name->Len());
+ topic_name.assign((const char *)BifConst::Kafka::topic_name->Bytes(),
+BifConst::Kafka::topic_name->Len());
// kafka_conf - thread local copy
- Val* val = BifConst::Kafka::kafka_conf->AsTableVal();
- IterCookie* c = val->AsTable()->InitForIteration();
- HashKey* k;
- TableEntryVal* v;
+ Val *val = BifConst::Kafka::kafka_conf->AsTableVal();
+ IterCookie *c = val->AsTable()->InitForIteration();
+ HashKey *k;
+ TableEntryVal *v;
while ((v = val->AsTable()->NextEntry(k, c))) {
+// fetch the key and value
+ListVal *index = val->AsTableVal()->RecoverIndex(k);
+std::string key = index->Index(0)->AsString()->CheckString();
+std::string val = v->Value()->AsString()->CheckString();
+kafka_conf.insert(kafka_conf.begin(),
+ std::pair(key, val));
- // fetch the key and value
- ListVal* index = val->AsTableVal()->RecoverIndex(k);
- string key = index->Index(0)->AsString()->CheckString();
- string val = v->Value()->AsString()->CheckString();
- kafka_conf.insert (kafka_conf.begin(), pair (key, val));
-
- // cleanup
- Unref(index);
- delete k;
+// cleanup
+Unref(index);
+delete k;
}
- Val* mvals = BifConst::Kafka::additional_message_values->AsTableVal();
+ Val *mvals = BifConst::Kafka::additional_message_values->AsTableVal();
c = val->AsTable()->InitForIteration();
while ((v = mvals->AsTable()->NextEntry(k, c))) {
-
-// fetch the key and value
-ListVal* index = mvals->AsTableVal()->RecoverIndex(k);
-string key = index->Index(0)->AsString()->CheckString();
-string val = v->Value()->
[metron] branch master updated: METRON-2358 Fix typos in descriptions in metron analytics module (LeeJiangchuan via ottobackwards) closes apache/metron#1582
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 1252573 METRON-2358 Fix typos in descriptions in metron analytics
module (LeeJiangchuan via ottobackwards) closes apache/metron#1582
1252573 is described below
commit 1252573eab087e87fb2938f8651c5b0b393aca9e
Author: LeeJiangchuan
AuthorDate: Fri May 22 08:39:41 2020 -0400
METRON-2358 Fix typos in descriptions in metron analytics module
(LeeJiangchuan via ottobackwards) closes apache/metron#1582
---
.../apache/metron/profiler/client/window/predicates/DayPredicates.java | 2 +-
.../main/java/org/apache/metron/profiler/storm/ProfileHBaseMapper.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/predicates/DayPredicates.java
b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/predicates/DayPredicates.java
index 2c5adb4..9a0e9d3 100644
---
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/predicates/DayPredicates.java
+++
b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/predicates/DayPredicates.java
@@ -77,7 +77,7 @@ public enum DayPredicates {
*/
HOLIDAY(x -> new HolidaysPredicate(x)),
/**
- * True if the day is a specifie ddate, false otherwise.
+ * True if the day is a specified date, false otherwise.
*/
DATE( x -> new DateSpecifierPredicate(x))
;
diff --git
a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileHBaseMapper.java
b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileHBaseMapper.java
index bbb5860..2ba213f 100644
---
a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileHBaseMapper.java
+++
b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileHBaseMapper.java
@@ -44,7 +44,7 @@ public class ProfileHBaseMapper implements HBaseMapper {
private RowKeyBuilder rowKeyBuilder;
/**
- * Generates the ColumnList necesary to store profile data in HBase.
+ * Generates the ColumnList necessary to store profile data in HBase.
*/
private ColumnBuilder columnBuilder;
[metron-bro-plugin-kafka] branch master updated: METRON-2343 Bro Kafka plugin - ability to dynamically modify JSON (ottobackwards) closes apache/metron-bro-plugin-kafka#46
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new 8da1637 METRON-2343 Bro Kafka plugin - ability to dynamically modify
JSON (ottobackwards) closes apache/metron-bro-plugin-kafka#46
8da1637 is described below
commit 8da1637a50815d6093e482bdb7a1a0882e02df3a
Author: ottobackwards
AuthorDate: Tue May 19 20:54:23 2020 -0400
METRON-2343 Bro Kafka plugin - ability to dynamically modify JSON
(ottobackwards) closes apache/metron-bro-plugin-kafka#46
---
README.md| 28
docker/in_docker_scripts/configure_plugin.sh | 2 ++
scripts/init.zeek| 7 +++
src/KafkaWriter.cc | 28 +++-
src/KafkaWriter.h| 1 +
src/TaggedJSON.cc| 22 +++---
src/TaggedJSON.h | 2 +-
src/kafka.bif| 1 +
tests/Baseline/kafka.show-plugin/output | 1 +
9 files changed, 83 insertions(+), 9 deletions(-)
diff --git a/README.md b/README.md
index b4aa98d..b7e7e58 100644
--- a/README.md
+++ b/README.md
@@ -266,6 +266,22 @@ event zeek_init() =-10
_Note_: Because `Kafka::tag_json` is set to True in this example, the value
of `$path` is used as the tag for each `Log::Filter`. If you were to add a log
filter with the same `$path` as an existing filter, Zeek will append "-N",
where N is an integer starting at 2, to the end of the log path so that each
filter has its own unique log path. For instance, the second instance of `conn`
would become `conn-2`.
+### Example 7 - Add static values to each outgoing Kafka message
+It is possible to define name value pairs and have them added to each outgoing
Kafka json message when tagged_json is set to true. Each will be added to the
root json object.
+* the Kafka::additional_message_values table can be configured with each
name and value
+* based on the following configuration, each outgoing message will have
"FIRST_STATIC_NAME": "FIRST_STATIC_VALUE", "SECOND_STATIC_NAME":
"SECOND_STATIC_VALUE" added.
+```
+@load packages
+redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG, Conn::LOG, DPD::LOG,
FTP::LOG, Files::LOG, Known::CERTS_LOG, SMTP::LOG, SSL::LOG, Weird::LOG,
Notice::LOG, DHCP::LOG, SSH::LOG, Software::LOG, RADIUS::LOG, X509::LOG,
RFB::LOG, Stats::LOG, CaptureLoss::LOG, SIP::LOG);
+redef Kafka::topic_name = "zeek";
+redef Kafka::tag_json = T;
+redef Kafka::kafka_conf = table(["metadata.broker.list"] =
"kafka-1:9092,kafka-2:9092");
+redef Kafka::additional_message_values = table(["FIRST_STATIC_NAME"] =
"FIRST_STATIC_VALUE", ["SECOND_STATIC_NAME"] = "SECOND_STATIC_VALUE");
+redef Kafka::logs_to_exclude = set(Conn::LOG, DHCP::LOG);
+redef Known::cert_tracking = ALL_HOSTS;
+redef Software::asset_tracking = ALL_HOSTS;
+```
+
## Settings
### `logs_to_send`
@@ -315,6 +331,18 @@ redef Kafka::kafka_conf = table(
);
```
+### `additonal_message_values`
+
+A table of of name value pairs. Each item in this table will be added to each
outgoing message
+at the root level if tag_json is set to T.
+
+```
+redef Kafka::additional_message_values = table(
+["FIRST_STATIC_NAME"] = "FIRST_STATIC_VALUE",
+["SECOND_STATIC_NAME"] = "SECOND_STATIC_VALUE"
+);
+```
+
### `tag_json`
If true, a log stream identifier is appended to each JSON-formatted message.
For
diff --git a/docker/in_docker_scripts/configure_plugin.sh
b/docker/in_docker_scripts/configure_plugin.sh
index c4479db..8d2f3da 100755
--- a/docker/in_docker_scripts/configure_plugin.sh
+++ b/docker/in_docker_scripts/configure_plugin.sh
@@ -23,6 +23,7 @@ shopt -s nocasematch
# Configures the zeek kafka plugin
# Configures the kafka broker
# Configures the plugin for all the traffic types
+# Configures the plugin to add some additional json values
#
function help {
@@ -74,6 +75,7 @@ echo "Configuring kafka plugin"
echo "redef Kafka::topic_name = \"${KAFKA_TOPIC}\";"
echo "redef Kafka::tag_json = T;"
echo "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] =
\"kafka-1:9092,kafka-2:9092\");"
+ echo "redef Kafka::additional_message_values = table([\"FIRST_STATIC_NAME\"]
= \"FIRST_STATIC_VALUE\", [\"SECOND_STATIC_NAME\"] = \"SECOND_STATIC_VALUE\");"
echo "redef Kafka::logs_to_exclude = set(Conn::LOG, DHCP::LOG);"
echo "redef Known::cert_tracking = ALL_HOSTS;"
echo "redef Software::ass
[metron-bro-plugin-kafka] branch master updated: METRON-2353 Plugin e2e tests should support multiple brokers (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#45
This is an automated email from the ASF dual-hosted git repository. otto pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git The following commit(s) were added to refs/heads/master by this push: new 853285c METRON-2353 Plugin e2e tests should support multiple brokers (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#45 853285c is described below commit 853285cca8bcac6927dc983151fb03a1a2062cc9 Author: JonZeolla AuthorDate: Mon May 18 18:28:17 2020 -0400 METRON-2353 Plugin e2e tests should support multiple brokers (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#45 --- .gitignore | 136 - README.md | 16 +-- docker/README.md | 29 +++-- docker/containers/kafka/Dockerfile | 11 +- docker/containers/zeek/Dockerfile | 37 +++--- docker/containers/zeek/requirements.txt| 6 +- docker/containers/zookeeper/Dockerfile | 7 +- docker/docker-compose.yml | 30 +++-- docker/finish_end_to_end.sh| 1 + docker/in_docker_scripts/build_plugin.sh | 2 +- docker/in_docker_scripts/configure_plugin.sh | 2 +- docker/in_docker_scripts/process_data_file.sh | 8 +- docker/remove_timeout_message.sh | 51 docker/run_end_to_end.sh | 110 - docker/scripts/docker_execute_build_plugin.sh | 5 +- docker/scripts/docker_execute_configure_plugin.sh | 3 - .../docker_execute_configure_zeek_plugin.sh| 94 -- .../docker_execute_create_topic_in_kafka.sh| 24 ++-- docker/scripts/docker_execute_process_data_file.sh | 5 +- docker/scripts/docker_run_consume_kafka.sh | 24 ++-- docker/scripts/docker_run_get_offset_kafka.sh | 8 +- docker/scripts/split_kafka_output_by_log.sh| 21 ++-- src/KafkaWriter.cc | 2 +- src/KafkaWriter.h | 1 - .../output | 2 +- .../kafka/resolved-topic-override-and-config.zeek | 4 +- 26 files changed, 323 insertions(+), 316 deletions(-) diff --git a/.gitignore b/.gitignore index e08b80e..2fd1f45 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,22 @@ -#ide stuff +# ide stuff .idea *.iml *.iws -.DS_Store /cmake-build-* .state build +# Log files +*.log + + +# Created by https://www.gitignore.io/api/vim,c++,emacs,git,macos +# Edit at https://www.gitignore.io/?templates=vim,c++,emacs,git,macos + +### C++ ### +# Prerequisites +*.d + # Compiled Object files *.slo *.lo @@ -24,6 +34,7 @@ build # Fortran module files *.mod +*.smod # Compiled Static libraries *.lai @@ -36,5 +47,122 @@ build *.out *.app -# Log files -*.log +### Emacs ### +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# Org-mode +.org-id-locations +*_archive + +# flymake-mode +*_flymake.* + +# eshell files +/eshell/history +/eshell/lastdir + +# elpa packages +/elpa/ + +# reftex files +*.rel + +# AUCTeX auto folder +/auto/ + +# cask packages +.cask/ +dist/ + +# Flycheck +flycheck_*.el + +# server auth directory +/server/ + +# projectiles files +.projectile + +# directory configuration +.dir-locals.el + +# network security +/network-security.data + + +### Git ### +# Created by git for backups. To disable backups in Git: +# $ git config --global mergetool.keepBackup false +*.orig + +# Created by git when using merge tools for conflicts +*.BACKUP.* +*.BASE.* +*.LOCAL.* +*.REMOTE.* +*_BACKUP_*.txt +*_BASE_*.txt +*_LOCAL_*.txt +*_REMOTE_*.txt + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist + +# Auto-generated tag files +tags + +# Persistent undo +[._]*.un~ + +# Coc configuration directory +.vim + +# End of https://www.gitignore.io/api/vim,c++,emacs,git,macos diff --git a/README.md b/README.md index 054ca04..b4aa98d 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,13 @@ This software is a part of the [Apache Metron](https://metron.apache.org/) proje `zkg` is the preferred mechanism for installing this plugin, as it will dynamically retrieve, build, test, and load the plugin. Note
[metron] branch master updated: METRON-2348 prepare-commit broken by github api change (ottobackwards) closes apache/metron#1580
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 8fa2566 METRON-2348 prepare-commit broken by github api change
(ottobackwards) closes apache/metron#1580
8fa2566 is described below
commit 8fa25664fdff439ba28492bf43e26788691ec84b
Author: ottobackwards
AuthorDate: Sun May 17 21:31:07 2020 -0400
METRON-2348 prepare-commit broken by github api change (ottobackwards)
closes apache/metron#1580
---
dev-utilities/committer-utils/README.md | 2 ++
dev-utilities/committer-utils/metron-committer-common | 12 +---
2 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/dev-utilities/committer-utils/README.md
b/dev-utilities/committer-utils/README.md
index 0a91759..f0f358f 100644
--- a/dev-utilities/committer-utils/README.md
+++ b/dev-utilities/committer-utils/README.md
@@ -28,6 +28,8 @@ When prompted the `[value in brackets]` is used by default.
To accept the defau
In the following example, I enter the pull request number when prompted.
Using the pull request number, the script can extract most of the remaining
required information.
+> prepare-commit utilizes [jq](https://stedolan.github.io/jq/) to parse the
github api returns, so it is required
+
1. Execute the script.
The first time the script is run, you will be prompted for additional
information including your Apache username, Apache email, and Github username.
These values are persisted in `~/.metron-prepare-commit`. Subsequent
executions of the script will retrieve these values, rather than prompting you
again for them.
diff --git a/dev-utilities/committer-utils/metron-committer-common
b/dev-utilities/committer-utils/metron-committer-common
index 257bcb0..88dc576 100644
--- a/dev-utilities/committer-utils/metron-committer-common
+++ b/dev-utilities/committer-utils/metron-committer-common
@@ -128,9 +128,15 @@ function read_pull_request {
fi
# ensure that the pull request exists
-PR_EXISTS=$(curl -sI
https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/${PR} | grep Status: |
sed 's/[^0-9]//g')
-if [[ "$PR_EXISTS" != "200" ]]; then
- echo "Error: pull request #$PR does not exist"
+PR_STATE=$(curl -s
https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/${PR} | jq ".state")
+PR_MERGEABLE=$(curl -s
https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/${PR} | jq
".mergeable_state")
+if [[ "$PR_STATE" != "\"open\"" ]]; then
+ echo "Error: pull request #$PR is not open or does not exist"
+ exit 1
+fi
+
+if [[ "$PR_MERGEABLE" != "\"clean\"" ]] && [[ "$PR_MERGEABLE" !=
"\"unstable\"" ]]; then
+ echo "Error: pull request #$PR is not mergable. mergeable_state is
$PR_MERGEABLE"
exit 1
fi
}
[metron-bro-plugin-kafka] branch master updated: METRON-2346 Update kafka plugin testing dependencies (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#41
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new 1772b1f METRON-2346 Update kafka plugin testing dependencies
(JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#41
1772b1f is described below
commit 1772b1fabb50e0710eafe93c14ec2d678c0a8bb9
Author: JonZeolla
AuthorDate: Mon Apr 27 09:34:49 2020 -0400
METRON-2346 Update kafka plugin testing dependencies (JonZeolla via
ottobackwards) closes apache/metron-bro-plugin-kafka#41
---
docker/README.md | 4 ++--
docker/containers/bro-localbuild-container/Dockerfile | 6 --
docker/scripts/docker_run_zookeeper_container.sh | 2 +-
docker/scripts/download_sample_pcaps.sh | 4 ++--
4 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/docker/README.md b/docker/README.md
index dbfbd89..c82e247 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -188,9 +188,9 @@ testing scripts to be added to a pull request, and
subsequently to a test suite.
- `download_sample_pcaps.sh`: Downloads the sample pcaps to a specified
directory. If they exist, it is a no-op
> The sample pcaps are:
- > - https://www.bro.org/static/traces/exercise-traffic.pcap
+ > -
https://github.com/bro/try-bro/blob/master/manager/static/pcaps/exercise_traffic.pcap
> -
http://downloads.digitalcorpora.org/corpora/network-packet-dumps/2008-nitroba/nitroba.pcap
- > - https://www.bro.org/static/traces/ssh.pcap
+ > -
https://github.com/bro/try-bro/raw/master/manager/static/pcaps/ssh.pcap
> -
https://github.com/markofu/pcaps/blob/master/PracticalPacketAnalysis/ppa-capture-files/ftp.pcap?raw=true
> -
https://github.com/EmpowerSecurityAcademy/wireshark/blob/master/radius_localhost.pcapng?raw=true
> - https://github.com/kholia/my-pcaps/blob/master/VNC/07-vnc
diff --git a/docker/containers/bro-localbuild-container/Dockerfile
b/docker/containers/bro-localbuild-container/Dockerfile
index b35724e..9b34baa 100644
--- a/docker/containers/bro-localbuild-container/Dockerfile
+++ b/docker/containers/bro-localbuild-container/Dockerfile
@@ -30,8 +30,10 @@ RUN yum -y groupinstall "Development Tools" && \
COPY .screenrc /root
# install bro
-RUN curl -L https://www.bro.org/downloads/bro-2.5.5.tar.gz | tar xvz
-WORKDIR bro-2.5.5/
+RUN git clone https://github.com/bro/bro
+WORKDIR bro/
+RUN git checkout v2.5.5
+RUN git submodule update --init --recursive
RUN ./configure
RUN make
RUN make install
diff --git a/docker/scripts/docker_run_zookeeper_container.sh
b/docker/scripts/docker_run_zookeeper_container.sh
index c6024b0..b35b813 100755
--- a/docker/scripts/docker_run_zookeeper_container.sh
+++ b/docker/scripts/docker_run_zookeeper_container.sh
@@ -78,5 +78,5 @@ rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc}
fi
-echo "Started the zookeeper container with networ ${NETWORK_NAME}"
+echo "Started the zookeeper container with network ${NETWORK_NAME}"
diff --git a/docker/scripts/download_sample_pcaps.sh
b/docker/scripts/download_sample_pcaps.sh
index 6a12af3..408dcac 100755
--- a/docker/scripts/download_sample_pcaps.sh
+++ b/docker/scripts/download_sample_pcaps.sh
@@ -87,7 +87,7 @@ for folder in nitroba example-traffic ssh ftp radius rfb; do
done
if [[ ! -f "${DATA_PATH}"/example-traffic/exercise-traffic.pcap ]]; then
- wget https://www.bro.org/static/traces/exercise-traffic.pcap -O
"${DATA_PATH}"/example-traffic/exercise-traffic.pcap
+ wget
https://github.com/bro/try-bro/raw/master/manager/static/pcaps/exercise_traffic.pcap
-O "${DATA_PATH}"/example-traffic/exercise-traffic.pcap
fi
if [[ ! -f "${DATA_PATH}"/nitroba/nitroba.pcap ]]; then
@@ -95,7 +95,7 @@ if [[ ! -f "${DATA_PATH}"/nitroba/nitroba.pcap ]]; then
fi
if [[ ! -f "${DATA_PATH}"/ssh/ssh.pcap ]]; then
- wget https://www.bro.org/static/traces/ssh.pcap -O
"${DATA_PATH}"/ssh/ssh.pcap
+ wget https://github.com/bro/try-bro/raw/master/manager/static/pcaps/ssh.pcap
-O "${DATA_PATH}"/ssh/ssh.pcap
fi
if [[ ! -f "${DATA_PATH}"/ftp/ftp.pcap ]]; then
[metron-bro-plugin-kafka] branch master updated: METRON-2288 [BRO-KAFKA-PLUGIN] Should have it s own RC scripts (ottobackwards) closes apache/metron-bro-plugin-kafka#38
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new 8dd10b8 METRON-2288 [BRO-KAFKA-PLUGIN] Should have its own RC
scripts (ottobackwards) closes apache/metron-bro-plugin-kafka#38
8dd10b8 is described below
commit 8dd10b88077c139cedf1edd395630b9eaf4e05af
Author: ottobackwards
AuthorDate: Thu Feb 27 09:58:34 2020 -0500
METRON-2288 [BRO-KAFKA-PLUGIN] Should have its own RC scripts
(ottobackwards) closes apache/metron-bro-plugin-kafka#38
---
.../release-utils/metron-bro-kafka-rc-check| 224 +
1 file changed, 224 insertions(+)
diff --git a/dev_utilities/release-utils/metron-bro-kafka-rc-check
b/dev_utilities/release-utils/metron-bro-kafka-rc-check
new file mode 100755
index 000..edecaca
--- /dev/null
+++ b/dev_utilities/release-utils/metron-bro-kafka-rc-check
@@ -0,0 +1,224 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+shopt -s nocasematch
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo "-v/--version= The version of the metron bro plugin
kafka release. [Required]"
+ echo "-c/--candidate= Defines the Release Candidate.
[Required]"
+ echo "-h/--helpUsage information."
+ echo " "
+ echo "example: "
+ echo "metron-bro-kafka-rc-check --version=0.3.0 --candidate=RC2"
+ echo " "
+}
+
+APACHE_REPO="https://dist.apache.org/repos/dist/;
+METRON_DIST=${APACHE_REPO}"dev/metron/metron-bro-plugin-kafka/"
+METRON_KEYS=${APACHE_REPO}"release/metron/KEYS"
+
+#
+# runs the bro kafka plugin's docker based tests
+#
+function run_bro_docker {
+cd docker &> /dev/null || { echo "failed to change directory to docker" ;
exit 1; }
+./run_end_to_end.sh
+
+rc=$?; if [[ ${rc} != 0 ]]; then
+echo "ERROR> FAILED run_end_to_end"
+# do NOT exit here
+fi
+cd .. &> /dev/null || { echo "failed to change directory to plugin root";
exit 1; }
+}
+
+#
+# runs the finish bro docker script to cleanup
+#
+function finish_bro_docker {
+cd docker &> /dev/null || { echo "failed to change directory to docker";
exit 1; }
+./finish_end_to_end.sh
+
+rc=$?; if [[ ${rc} != 0 ]]; then
+echo "ERROR> FAILED finish_end_to_end"
+exit ${rc}
+fi
+cd .. &> /dev/null || { echo "failed to change directory to plugin root";
+exit 1; }
+}
+
+# print help, if the user just runs this without any args
+if [ "$#" -eq 0 ]; then
+help
+exit 1
+fi
+
+# handle command line options
+for i in "$@"; do
+ case $i in
+#
+# VERSION: The release version of Metron to validate.
+#
+#
+-v=*|--version=*)
+VERSION="${i#*=}"
+shift # past argument=value
+;;
+
+#
+# RC: Defines the RC# to use
+#
+# -c=RC2
+# --candidate=RC2
+#
+-c=*|--candidate=*)
+CANDIDATE="${i#*=}"
+shift # past argument=value
+;;
+
+#
+# -h/--help
+#
+-h|--help)
+help
+exit 0
+shift # past argument with no value
+;;
+
+#
+# Unknown option
+#
+*)
+UNKNOWN_OPTION="${i#*=}"
+echo "Error: unknown option: $UNKNOWN_OPTION"
+help
+;;
+ esac
+done
+
+# validation
+if [ -z "$VERSION" ]; then
+ echo "Missing -v/--version is is required"
+ exit 1
+fi
+if [[ "$VERSION" =~ ^[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2} ]]; then
+ PLUGIN_VERSION="$VERSION"
+else
+ echo "[ERROR] "$VERSION" may not be a valid version number"
+ exit 1
+fi
+
+if [ -z "$CANDIDATE" ]; then
+ echo "Missing -c/--candidate which is required"
+ exit 1
+fi
+
+if [[ "$CANDIDATE" =~ ^RC[0-9]+ ]]; then
+ RC=$(echo "$CANDIDATE" | tr '[:upper:]'
[metron-bro-plugin-kafka] branch master updated: METRON-2269 Cannot run Docker tests if src is not a git repo (ottobackwards) closes apache/metron-bro-plugin-kafka#37
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new abbbc9b METRON-2269 Cannot run Docker tests if src is not a git repo
(ottobackwards) closes apache/metron-bro-plugin-kafka#37
abbbc9b is described below
commit abbbc9b8e049fa281dde1ff6ab159d194dc10d4a
Author: ottobackwards
AuthorDate: Tue Oct 1 14:56:01 2019 -0400
METRON-2269 Cannot run Docker tests if src is not a git repo
(ottobackwards) closes apache/metron-bro-plugin-kafka#37
---
docker/run_end_to_end.sh | 35 ---
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
index 9a7036f..6d003b0 100755
--- a/docker/run_end_to_end.sh
+++ b/docker/run_end_to_end.sh
@@ -19,7 +19,6 @@
shopt -s nocasematch
set -u # nounset
-set -e # errexit
set -E # errtrap
set -o pipefail
@@ -48,6 +47,7 @@ fi
SKIP_REBUILD_BRO=false
NO_PCAP=false
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+PLUGIN_ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. > /dev/null &&
pwd)"
SCRIPT_DIR="${ROOT_DIR}"/scripts
CONTAINER_DIR="${ROOT_DIR}"/containers/bro-localbuild-container
DATA_PATH="${ROOT_DIR}"/data
@@ -55,7 +55,36 @@ DATE=$(date)
LOG_DATE=${DATE// /_}
TEST_OUTPUT_PATH="${ROOT_DIR}/test_output/"${LOG_DATE//:/_}
KAFKA_TOPIC="bro"
-PLUGIN_VERSION=$(cd "${ROOT_DIR}" && git rev-parse --symbolic-full-name
--abbrev-ref HEAD)
+
+cd "${PLUGIN_ROOT_DIR}" || { echo "NO PLUGIN ROOT" ; exit 1; }
+# we may not be checked out from git, check and make it so that we are since
+# bro-pkg requires it
+
+git status 2&>1
+rc=$?; if [[ ${rc} != 0 ]]; then
+ echo "bro-pkg requires the plugin to be a git repo, creating..."
+ git init .
+ rc=$?; if [[ ${rc} != 0 ]]; then
+echo "ERROR> FAILED TO INITIALIZE GIT IN PLUGIN DIRECTORY. ${rc}"
+ exit ${rc}
+ fi
+ git add .
+ rc=$?; if [[ ${rc} != 0 ]]; then
+echo "ERROR> FAILED TO ADD ALL TO GIT PLUGIN DIRECTORY. ${rc}"
+ exit ${rc}
+ fi
+ git commit -m 'docker run'
+ rc=$?; if [[ ${rc} != 0 ]]; then
+echo "ERROR> FAILED TO COMMIT TO GIT MASTER IN PLUGIN DIRECTORY. ${rc}"
+ exit ${rc}
+ fi
+ echo "git repo created"
+fi
+
+# set errexit for the rest of the run
+set -e
+
+PLUGIN_VERSION=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD)
# Handle command line options
for i in "$@"; do
@@ -119,7 +148,7 @@ for i in "$@"; do
done
EXTRA_ARGS="$*"
-
+cd "${ROOT_DIR}" || { echo "NO ROOT" ; exit 1; }
echo "Running build_container with "
echo "SKIP_REBUILD_BRO = ${SKIP_REBUILD_BRO}"
echo "DATA_PATH= ${DATA_PATH}"
[metron-bro-plugin-kafka] branch master updated: METRON-2069 Add btests for bro plugin topic_name selection (ottobackwards) closes apache/metron-bro-plugin-kafka#36
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new e70500c METRON-2069 Add btests for bro plugin topic_name selection
(ottobackwards) closes apache/metron-bro-plugin-kafka#36
e70500c is described below
commit e70500cd178e70dfe49c6f00c3d42f4b8eed2fea
Author: ottobackwards
AuthorDate: Wed Sep 25 09:38:24 2019 -0400
METRON-2069 Add btests for bro plugin topic_name selection (ottobackwards)
closes apache/metron-bro-plugin-kafka#36
---
.gitignore | 5 +-
CMakeLists.txt | 1 +
docker/README.md | 1 +
.../containers/bro-localbuild-container/Dockerfile | 2 +-
docker/in_docker_scripts/build_bro_plugin.sh | 19 +++-
docker/run_end_to_end.sh | 88 +++---
...tput_by_log.sh => split_kafka_output_by_log.sh} | 0
scripts/Apache/Kafka/logs-to-kafka.bro | 5 +
scripts/init.bro | 2 +
src/KafkaWriter.cc | 126 +
src/KafkaWriter.h | 2 +
src/events.bif | 1 +
src/kafka.bif | 1 +
tests/Baseline/kafka.resolved-topic-config/output | 1 +
tests/Baseline/kafka.resolved-topic-default/output | 1 +
.../output | 2 +
.../kafka.resolved-topic-override-only/output | 1 +
tests/Baseline/kafka.show-plugin/output| 2 +
.../kafka/resolved-topic-config.bro| 13 ++-
.../kafka/resolved-topic-default.bro | 11 +-
.../kafka/resolved-topic-override-and-config.bro | 24 +++-
.../kafka/resolved-topic-override-only.bro | 21 +++-
tests/pcaps/exercise-traffic.pcap | Bin 0 -> 83264745 bytes
23 files changed, 205 insertions(+), 124 deletions(-)
diff --git a/.gitignore b/.gitignore
index 735a17e..e08b80e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,7 +3,7 @@
*.iml
*.iws
.DS_Store
-
+/cmake-build-*
.state
build
@@ -38,6 +38,3 @@ build
# Log files
*.log
-# pcap files
-*.pcap
-*.pcapng
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2bc8d4b..18b8a14 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -28,6 +28,7 @@ if (LIBRDKAFKA_FOUND AND OPENSSL_FOUND)
bro_plugin_cc(src/Plugin.cc)
bro_plugin_cc(src/TaggedJSON.cc)
bro_plugin_bif(src/kafka.bif)
+ bro_plugin_bif(src/events.bif)
bro_plugin_dist_files(README CHANGES COPYING VERSION)
bro_plugin_link_library(${LibRDKafka_LIBRARIES})
bro_plugin_link_library(${LibRDKafka_C_LIBRARIES})
diff --git a/docker/README.md b/docker/README.md
index eac2919..dbfbd89 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -295,6 +295,7 @@ Other scripts may then be used to do your testing, for
example running:
## Parameters
```bash
--skip-docker-build [OPTIONAL] Skip build of bro docker machine.
+--no-pcaps [OPTIONAL] Do not run pcaps.
--data-path [OPTIONAL] The pcap data path. Default: ./data
--kafka-topic [OPTIONAL] The kafka topic name to use.
Default: bro
--plugin-version[OPTIONAL] The plugin version. Default: the
current branch name
diff --git a/docker/containers/bro-localbuild-container/Dockerfile
b/docker/containers/bro-localbuild-container/Dockerfile
index 6a881f5..b35724e 100644
--- a/docker/containers/bro-localbuild-container/Dockerfile
+++ b/docker/containers/bro-localbuild-container/Dockerfile
@@ -24,7 +24,7 @@ RUN yum -y groupinstall "Development Tools" && \
openssl-devel python-devel swig \
zlib-devel perl \
cyrus-sasl cyrus-sasl-devel cyrus-sasl-gssapi \
- git jq screen
+ git jq screen tree vim
# copy in the screen -rc
COPY .screenrc /root
diff --git a/docker/in_docker_scripts/build_bro_plugin.sh
b/docker/in_docker_scripts/build_bro_plugin.sh
index 064fe91..401b2a5 100755
--- a/docker/in_docker_scripts/build_bro_plugin.sh
+++ b/docker/in_docker_scripts/build_bro_plugin.sh
@@ -82,19 +82,26 @@ echo "PLUGIN_VERSION = ${PLUGIN_VERSION}"
cd /root || exit 1
-echo ""
+echo "==="
-bro-pkg install code --version "${PLUGIN_VERSION}" --force
+bro-pkg -vvv install code --version "${PLUGIN_VERSION}" --force
rc=$?; if [[ ${rc} != 0 ]]; then
echo "ERROR running bro-pkg install ${rc}"
exit ${rc}
fi
-
-echo ""
+echo "==="
+echo "ERR"
+cat /root/.zkg/testing/code/clon
[metron] branch master updated: METRON-2185 Update Simple-Syslog dependency to fix error in Structured Data (ottobackwards) closes apache/metron#1466
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 43f74fc METRON-2185 Update Simple-Syslog dependency to fix error in
Structured Data (ottobackwards) closes apache/metron#1466
43f74fc is described below
commit 43f74fc91a24ea2b24f5a41038cdf46d3d883fa1
Author: ottobackwards
AuthorDate: Fri Jul 19 10:10:04 2019 -0400
METRON-2185 Update Simple-Syslog dependency to fix error in Structured Data
(ottobackwards) closes apache/metron#1466
---
dependencies_with_url.csv | 2 +-
.../src/main/sample/data/syslog5424/parsed/Syslog5424Parsed | 4 ++--
.../src/main/sample/data/syslog5424/raw/Syslog5424Output| 4 ++--
.../java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java | 6 +++---
pom.xml | 2 +-
5 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 04f7935..595cf0e 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -494,7 +494,7 @@ org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile
org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
com.zaxxer:HikariCP:jar:2.7.8:compile,ASLv2,https://github.com/brettwooldridge/HikariCP
org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https://github.com/hibernate/hibernate-validator
-com.github.palindromicity:simple-syslog:jar:0.0.1:compile,ASLv2,https://github.com/palindromicity/simple-syslog
+com.github.palindromicity:simple-syslog:jar:0.0.3:compile,ASLv2,https://github.com/palindromicity/simple-syslog
org.elasticsearch.client:elasticsearch-rest-high-level-client:jar:5.6.14:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
org.elasticsearch.plugin:aggs-matrix-stats-client:jar:5.6.14:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
org.fusesource.jansi:jansi:jar:1.16:compile,ASLv2,https://github.com/fusesource/jansi/blob/master/license.txt
diff --git
a/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
index ee1c6f6..82ebf15 100644
---
a/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
+++
b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
@@ -1,3 +1,3 @@
{"syslog.header.hostName":"loggregator","syslog.header.facility":"1","syslog.header.timestamp":"2014-06-20T09:14:07+00:00","syslog.message":"Removing
instance","syslog.header.pri":"14","syslog.header.procId":"DEA","syslog.header.severity":"6","source.type":"syslog5424","syslog.header.appName":"d0602076-b14a-4c55-852a-981e7afeed38","syslog.header.version":"1","original_string":"<14>1
2014-06-20T09:14:07+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed38 DEA
- - Removing instance","sys [...]
-{"syslog.structureddata.examples...@32480.iut":"4","syslog.structuredData.exampleSDID@32480.eventSource":"Other
Application","syslog.header.hostName":"loggregator","syslog.header.facility":"1","syslog.structuredData.exampleSDID@32480.eventID":"2022","syslog.structuredData.exampleSDID@32473.eventSource":"Application","syslog.header.timestamp":"2014-06-20T09:14:08+00:00","syslog.message":"Removing
instance","syslog.header.pri":"14","syslog.header.procId":"DEA","syslog.header.severity":"6",
[...]
-{"syslog.structureddata.examples...@32480.iut":"4","syslog.structuredData.exampleSDID@32480.eventSource":"Other
Application","syslog.structureddata.examples...@32474.iut":"3","syslog.structuredData.exampleSDID@32474.eventID":"1011","syslog.header.hostName":"loggregator","syslog.header.facility":"1","syslog.structuredData.exampleSDID@32480.eventID":"2022","syslog.header.timestamp":"2014-06-20T09:14:09+00:00","syslog.message":"Removing
instance","syslog.header.pri":"14","syslog.header.proc [...]
\ No newline at end of file
+{"syslog.structureddata.examples...@32480.iut":"4","syslog.structuredData.example
[metron] branch master updated: METRON-2018 Update prepare-commit to add Bro plugin tests (ottobackwards) closes apache/metron#1348
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new f3f5c97 METRON-2018 Update prepare-commit to add Bro plugin tests
(ottobackwards) closes apache/metron#1348
f3f5c97 is described below
commit f3f5c9743f515d8902b4580c765ad9d2b64aa63b
Author: ottobackwards
AuthorDate: Fri May 3 10:40:47 2019 -0400
METRON-2018 Update prepare-commit to add Bro plugin tests (ottobackwards)
closes apache/metron#1348
---
.../committer-utils/metron-committer-common| 43 +-
1 file changed, 41 insertions(+), 2 deletions(-)
diff --git a/dev-utilities/committer-utils/metron-committer-common
b/dev-utilities/committer-utils/metron-committer-common
index 4855254..257bcb0 100644
--- a/dev-utilities/committer-utils/metron-committer-common
+++ b/dev-utilities/committer-utils/metron-committer-common
@@ -343,8 +343,18 @@ function run_tests {
run_mvn_build_rpms
verify_licenses
+
elif [[ "${UPSTREAM}" == "${BRO_PLUGIN_UPSTREAM}" ]]; then
-echo "We don't currently support running metron-bro-plugin-kafka tests
in this script"
+echo " please verify that no bro docker containers are running before
continuing,"
+read -p " no bro docker containers are running, ready to proceed [yN]
" -n 1 -r
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]; then
+run_bro_docker
+finish_bro_docker
+else
+echo " when you are ready and the containers are stopped, please
cd into the docker"
+echo " directory and execute the run_end_to_end.sh script"
+fi
fi
fi
@@ -435,6 +445,35 @@ function run_mvn_build_rpms {
}
#
+# runs the bro kafka plugin's docker based tests
+#
+function run_bro_docker {
+cd docker &> /dev/null || { echo "failed to change directory to docker" ;
exit 1; }
+./run_end_to_end.sh
+
+rc=$?; if [[ ${rc} != 0 ]]; then
+echo "ERROR> FAILED run_end_to_end"
+# do NOT exit here
+fi
+cd .. &> /dev/null || { echo "failed to change directory to plugin root";
exit 1; }
+}
+
+#
+# runs the finish bro docker script to cleanup
+#
+function finish_bro_docker {
+cd docker &> /dev/null || { echo "failed to change directory to docker";
exit 1; }
+./finish_end_to_end.sh
+
+rc=$?; if [[ ${rc} != 0 ]]; then
+echo "ERROR> FAILED finish_end_to_end"
+exit ${rc}
+fi
+cd .. &> /dev/null || { echo "failed to change directory to plugin root";
+exit 1; }
+}
+
+#
# verifies licenses using our script
#
function verify_licenses {
@@ -455,4 +494,4 @@ function please_review_then {
echo "cd $WORK"
echo "git push upstream $BASE_BRANCH"
echo ""
-}
\ No newline at end of file
+}
[metron-bro-plugin-kafka] branch master updated: METRON-2025 Bro Kafka Plugin Docker should yum clean (ottobackwards) closes apache/metron-bro-plugin-kafka#33
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new 1d62215 METRON-2025 Bro Kafka Plugin Docker should yum clean
(ottobackwards) closes apache/metron-bro-plugin-kafka#33
1d62215 is described below
commit 1d6221531140bbe010590d107bf5890aa1cd9894
Author: ottobackwards
AuthorDate: Wed Mar 6 06:36:31 2019 -0500
METRON-2025 Bro Kafka Plugin Docker should yum clean (ottobackwards) closes
apache/metron-bro-plugin-kafka#33
---
docker/containers/bro-localbuild-container/Dockerfile | 17 -
1 file changed, 8 insertions(+), 9 deletions(-)
diff --git a/docker/containers/bro-localbuild-container/Dockerfile
b/docker/containers/bro-localbuild-container/Dockerfile
index d8ab804..6a881f5 100644
--- a/docker/containers/bro-localbuild-container/Dockerfile
+++ b/docker/containers/bro-localbuild-container/Dockerfile
@@ -36,17 +36,16 @@ RUN ./configure
RUN make
RUN make install
ENV PATH="${PATH}:/usr/local/bro/bin"
+ENV PATH="${PATH}:/usr/bin"
# install pip
-RUN yum -y update
-RUN yum -y install epel-release
-RUN yum -y install python-pip
-RUN pip install --upgrade pip
-
-# install bro-pkg
-RUN pip install bro-pkg
-ENV PATH="${PATH}:/usr/bin"
-RUN bro-pkg autoconfig
+RUN yum -y update && \
+yum -y install epel-release && \
+yum -y install python-pip && \
+yum clean all && \
+pip install --upgrade pip && \
+pip install bro-pkg && \
+bro-pkg autoconfig
# install librdkafka
RUN curl -L https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz |
tar xvz
[metron-bro-plugin-kafka] branch master updated: METRON-2021 Add screen to bro docker image (ottobackwards) closes apache/metron-bro-plugin-kafka#32
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new 3cbc785 METRON-2021 Add screen to bro docker image (ottobackwards)
closes apache/metron-bro-plugin-kafka#32
3cbc785 is described below
commit 3cbc78586a89c6c8c4385013b6ae63bdb8418b0e
Author: ottobackwards
AuthorDate: Sat Mar 2 09:31:36 2019 -0500
METRON-2021 Add screen to bro docker image (ottobackwards) closes
apache/metron-bro-plugin-kafka#32
---
.../containers/bro-localbuild-container/.screenrc | 22 ++
.../containers/bro-localbuild-container/Dockerfile | 12 ++--
2 files changed, 32 insertions(+), 2 deletions(-)
diff --git a/docker/containers/bro-localbuild-container/.screenrc
b/docker/containers/bro-localbuild-container/.screenrc
new file mode 100644
index 000..97750f5
--- /dev/null
+++ b/docker/containers/bro-localbuild-container/.screenrc
@@ -0,0 +1,22 @@
+# terminfo and termcap for nice 256 color terminal
+# allow bold colors - necessary for some reason
+attrcolor b ".I"
+
+# tell screen how to set colors. AB = background, AF=foreground
+termcapinfo xterm 'Co#256:AB=\E[48;5;%dm:AF=\E[38;5;%dm'
+
+# erase background with current bg color
+defbce "on"
+
+# the status at the bottom of the window
+hardstatus alwayslastline
+hardstatus string '%{gk}[ %{G}%H %{g}][%S][%= %{wk}%?%-Lw%?%{=b kR}(%{W}%n*%f
%t%?(%u)%?%{=b kR})%{= kw}%?%+Lw%?%?%= %{g}][%{Y}%l%{g}]%{=b C}[ %m/%d %c ]%{W}'
+
+#turn off the startup banner
+startup_message off
+
+#i want to see all screen messages for a longer time
+msgwait 86400
+
+# Set scrollback to 20k
+defscrollback 2
diff --git a/docker/containers/bro-localbuild-container/Dockerfile
b/docker/containers/bro-localbuild-container/Dockerfile
index f7ad66f..d8ab804 100644
--- a/docker/containers/bro-localbuild-container/Dockerfile
+++ b/docker/containers/bro-localbuild-container/Dockerfile
@@ -18,8 +18,16 @@ FROM centos:7
WORKDIR /root
# install development tools
-RUN yum -y groupinstall "Development Tools"
-RUN yum -y install cmake make gcc gcc-c++ flex bison libpcap libpcap-devel
openssl-devel python-devel swig zlib-devel perl cyrus-sasl cyrus-sasl-devel
cyrus-sasl-gssapi git jq
+RUN yum -y groupinstall "Development Tools" && \
+ yum -y install cmake make gcc gcc-c++ \
+ flex bison libpcap libpcap-devel \
+ openssl-devel python-devel swig \
+ zlib-devel perl \
+ cyrus-sasl cyrus-sasl-devel cyrus-sasl-gssapi \
+ git jq screen
+
+# copy in the screen -rc
+COPY .screenrc /root
# install bro
RUN curl -L https://www.bro.org/downloads/bro-2.5.5.tar.gz | tar xvz
[metron-bro-plugin-kafka] branch master updated: METRON-2013 The bro plugin docker script s topic name should be configurable (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new c497078 METRON-2013 The bro plugin docker scripts topic name
should be configurable (JonZeolla via ottobackwards) closes
apache/metron-bro-plugin-kafka#27
c497078 is described below
commit c49707874284dec11c37af138ffd99d503df9836
Author: JonZeolla
AuthorDate: Thu Feb 28 15:48:32 2019 -0500
METRON-2013 The bro plugin docker scripts topic name should be
configurable (JonZeolla via ottobackwards) closes
apache/metron-bro-plugin-kafka#27
---
docker/README.md | 18 --
docker/run_end_to_end.sh | 26 ++--
...et_bro_kafka.sh => docker_run_consume_kafka.sh} | 28 +++---
...afka.sh => docker_run_create_topic_in_kafka.sh} | 15 ++--
...bro_kafka.sh => docker_run_get_offset_kafka.sh} | 18 +++---
5 files changed, 77 insertions(+), 28 deletions(-)
diff --git a/docker/README.md b/docker/README.md
index 3bae93b..a965d8b 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -71,9 +71,9 @@ testing scripts to be added to a pull request, and
subsequently to a test suite.
├── docker_execute_process_data_file.sh
├── docker_execute_shell.sh
├── docker_run_bro_container.sh
-├── docker_run_consume_bro_kafka.sh
-├── docker_run_create_bro_topic_in_kafka.sh
-├── docker_run_get_offset_bro_topic_in_kafka.sh
+├── docker_run_consume_kafka.sh
+├── docker_run_create_topic_in_kafka.sh
+├── docker_run_get_offset_kafka.sh
├── docker_run_kafka_container.sh
├── docker_run_wait_for_kafka.sh
├── docker_run_wait_for_zookeeper.sh
@@ -146,22 +146,25 @@ testing scripts to be added to a pull request, and
subsequently to a test suite.
> You can then execute these scripts or use them together as part of testing
etc. by creating `docker execute` scripts like those here.
> The goal is to allow an individual to use and maintain their own library
of scripts to use instead of, or in concert with the scripts maintained by this
project.
-- `docker_run_consume_bro_kafka.sh`: Runs an instance of the kafka container,
with the console consumer `kafka-console-consumer.sh --topic bro --offset
$OFFSET --partition 0 --bootstrap-server kafka:9092`
+- `docker_run_consume_kafka.sh`: Runs an instance of the kafka container, with
the console consumer `kafka-console-consumer.sh --topic $KAFKA_TOPIC --offset
$OFFSET --partition 0 --bootstrap-server kafka:9092`
## Parameters
```bash
--network-name [OPTIONAL] The Docker network name. Default:
bro-network
--offset [OPTIONAL] The kafka offset. Default: -1
+ --kafka-topic [OPTIONAL] The kafka topic to consume from.
Default: bro
```
-- `docker_run_get_offset_bro_kafka.sh`: Runs an instance of the kafka
container and gets the current offset for the bro topic
+- `docker_run_get_offset_kafka.sh`: Runs an instance of the kafka container
and gets the current offset for the specified topic
## Parameters
```bash
--network-name [OPTIONAL] The Docker network name. Default:
bro-network
--offset [OPTIONAL] The kafka offset. Default: -1
+ --kafka-topic [OPTIONAL] The kafka topic to get the offset
from. Default: bro
```
-- `docker_run_create_bro_topic_in_kafka.sh`: Runs an instance of the kafka
container, creating the `bro` topic
+- `docker_run_create_topic_in_kafka.sh`: Runs an instance of the kafka
container, creating the specified topic
## Parameters
```bash
--network-name [OPTIONAL] The Docker network name. Default:
bro-network
+ --kafka-topic [OPTIONAL] The kafka topic to create.
Default: bro
```
- `docker_run_kafka_container.sh`: Runs the main instance of the kafka
container in the background
## Parameters
@@ -223,7 +226,7 @@ This script does the following:
3. Waits for zookeeper to be available
4. Runs the kafka container
5. Waits for kafka to be available
-6. Creates the bro topic
+6. Creates the specified topic
7. Downloads sample PCAP data
8. Runs the bro container in the background
@@ -294,4 +297,5 @@ Other scripts may then be used to do your testing, for
example running:
```bash
--skip-docker-build[OPTIONAL] Skip build of bro docker machine.
--data-path[OPTIONAL] The pcap data path. Default: ./data
+--kafka-topic [OPTIONAL] The kafka topic name to use.
Default: bro
```
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
index a1dfb6a..6dfd146 100755
--- a/docker/run_end_to_end.sh
+++ b/docker/run_end_to_end.sh
@@ -28,6 +28,7 @@ function help {
echo "USAGE"
echo "--skip-docker-bui
[metron-bro-plugin-kafka] branch master updated: METRON-2020 Running run_end_to_end.sh with docker give warning if bash 4.0 (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#31
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new 946e664 METRON-2020 Running run_end_to_end.sh with docker give
warning if bash 4.0 (JonZeolla via ottobackwards) closes
apache/metron-bro-plugin-kafka#31
946e664 is described below
commit 946e664b0a56fd99bfb505e976bd07ad02f5c733
Author: JonZeolla
AuthorDate: Thu Feb 28 14:43:21 2019 -0500
METRON-2020 Running run_end_to_end.sh with docker give warning if bash
4.0 (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#31
---
docker/run_end_to_end.sh | 11 ++-
docker/scripts/analyze_results.sh | 17 -
2 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
index ae06715..a1dfb6a 100755
--- a/docker/run_end_to_end.sh
+++ b/docker/run_end_to_end.sh
@@ -25,14 +25,23 @@ set -o pipefail
function help {
echo " "
- echo "usage: ${0}"
+ echo "USAGE"
echo "--skip-docker-build [OPTIONAL] Skip build of bro
docker machine."
echo "--data-path [OPTIONAL] The pcap data path.
Default: ./data"
echo "-h/--help Usage information."
echo " "
+ echo "COMPATABILITY"
+ echo " bash >= 4.0 is required."
echo " "
}
+# Require bash >= 4
+if (( BASH_VERSINFO[0] < 4 )); then
+ >&2 echo "ERROR> bash >= 4.0 is required" >&2
+ help
+ exit 1
+fi
+
SKIP_REBUILD_BRO=false
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
diff --git a/docker/scripts/analyze_results.sh
b/docker/scripts/analyze_results.sh
index 790ec18..01c026f 100755
--- a/docker/scripts/analyze_results.sh
+++ b/docker/scripts/analyze_results.sh
@@ -23,6 +23,12 @@ set -e # errexit
set -E # errtrap
set -o pipefail
+declare -r txtDEFAULT='\033[0m'
+# shellcheck disable=SC2034
+declare -r txtERROR='\033[0;31m'
+# shellcheck disable=SC2034
+declare -r txtWARN='\033[0;33m'
+
#
# Analyzes the results.csv files to identify issues
#
@@ -51,17 +57,18 @@ function _echo() {
esac
}
+# Require bash >= 4
+if (( BASH_VERSINFO[0] < 4 )); then
+ _echo ERROR "bash >= 4.0 is required"
+ exit 1
+fi
+
SCRIPT_NAME=$(basename -- "$0")
TEST_DIRECTORY=
declare -A LOGS_WITH_UNEQUAL_RESULTS
declare -a LOG_NAMES
declare -A OVERALL_LOG_CARDINALITY
declare -A LOG_ISSUE_COUNT
-declare -r txtDEFAULT='\033[0m'
-# shellcheck disable=SC2034
-declare -r txtERROR='\033[0;31m'
-# shellcheck disable=SC2034
-declare -r txtWARN='\033[0;33m'
# Handle command line options
for i in "$@"; do
[metron-bro-plugin-kafka] branch master updated: METRON-1991 Bro plugin docker scripts should exit nonzero when bro and kafka counts differ (JonZeolla via ottobackwards) closes apache/metron-bro-plugi
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new 2f1edcb METRON-1991 Bro plugin docker scripts should exit nonzero
when bro and kafka counts differ (JonZeolla via ottobackwards) closes
apache/metron-bro-plugin-kafka#29
2f1edcb is described below
commit 2f1edcbec745d110ff9489dbf03348e428ea4c76
Author: JonZeolla
AuthorDate: Thu Feb 28 12:29:55 2019 -0500
METRON-1991 Bro plugin docker scripts should exit nonzero when bro and
kafka counts differ (JonZeolla via ottobackwards) closes
apache/metron-bro-plugin-kafka#29
---
docker/README.md| 10 +-
docker/run_end_to_end.sh| 16 ++-
docker/scripts/analyze_results.sh | 207
docker/scripts/build_container.sh | 4 +-
docker/scripts/print_results.sh | 5 +-
docker/scripts/split_kakfa_output_by_log.sh | 3 +-
6 files changed, 237 insertions(+), 8 deletions(-)
diff --git a/docker/README.md b/docker/README.md
index 8e4d3fa..3bae93b 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -61,6 +61,7 @@ testing scripts to be added to a pull request, and
subsequently to a test suite.
Scripts executed on the host to setup and interact with the docker
containers
```bash
+├── analyze_results.sh
├── build_container.sh
├── cleanup_docker.sh
├── create_docker_network.sh
@@ -83,6 +84,11 @@ testing scripts to be added to a pull request, and
subsequently to a test suite.
└── stop_container.sh
```
+- `analyze_results.sh`: Analyzes the `results.csv` files for any issues
+ ## Parameters
+ ```bash
+ --test-directory [REQUIRED] The directory for the tests
+ ```
- `build_container.sh`: Runs docker build in the passed directory, and names
the results
## Parameters
```bash
@@ -191,12 +197,12 @@ testing scripts to be added to a pull request, and
subsequently to a test suite.
```bash
--data-path[REQUIRED] The pcap data path
```
-- `print_results.sh` : Prints the `results.csv` for all the pcaps processed in
the given directory to console
+- `print_results.sh`: Prints the `results.csv` for all the pcaps processed in
the given directory to console
## Parameters
```bash
--test-directory [REQUIRED] The directory for the tests
```
-- `split_kafka_output_by_log.sh` : For a pcap result directory, will create a
LOG.kafka.log for each LOG.log's entry in the kafka-output.log
+- `split_kafka_output_by_log.sh`: For a pcap result directory, will create a
LOG.kafka.log for each LOG.log's entry in the kafka-output.log
## Parameters
```bash
--log-directory[REQUIRED] The directory with the logs
diff --git a/docker/run_end_to_end.sh b/docker/run_end_to_end.sh
index 6baf679..ae06715 100755
--- a/docker/run_end_to_end.sh
+++ b/docker/run_end_to_end.sh
@@ -182,22 +182,34 @@ do
echo "OFFSET--> ${OFFSET}"
bash "${SCRIPT_DIR}"/docker_execute_process_data_file.sh
--pcap-file-name="${BASE_FILE_NAME}"
--output-directory-name="${DOCKER_DIRECTORY_NAME}"
-
rc=$?; if [[ ${rc} != 0 ]]; then
echo "ERROR> FAILED TO PROCESS ${file} DATA. CHECK LOGS, please run the
finish_end_to_end.sh when you are done."
exit ${rc}
fi
+
KAFKA_OUTPUT_FILE="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}/kafka-output.log"
bash "${SCRIPT_DIR}"/docker_run_consume_bro_kafka.sh --offset=$OFFSET |
"${ROOT_DIR}"/remove_timeout_message.sh | tee "${KAFKA_OUTPUT_FILE}"
-
rc=$?; if [[ ${rc} != 0 ]]; then
echo "ERROR> FAILED TO PROCESS ${DATA_PATH} DATA. CHECK LOGS"
fi
"${SCRIPT_DIR}"/split_kakfa_output_by_log.sh
--log-directory="${TEST_OUTPUT_PATH}/${DOCKER_DIRECTORY_NAME}"
+ rc=$?; if [[ ${rc} != 0 ]]; then
+echo "ERROR> ISSUE ENCOUNTERED WHEN SPLITTING KAFKA OUTPUT LOGS"
+ fi
done
"${SCRIPT_DIR}"/print_results.sh --test-directory="${TEST_OUTPUT_PATH}"
+rc=$?; if [[ ${rc} != 0 ]]; then
+ echo "ERROR> ISSUE ENCOUNTERED WHEN PRINTING RESULTS"
+ exit ${rc}
+fi
+
+"${SCRIPT_DIR}"/analyze_results.sh --test-directory="${TEST_OUTPUT_PATH}"
+rc=$?; if [[ ${rc} != 0 ]]; then
+ echo "ERROR> ISSUE ENCOUNTERED WHEN ANALYZING RESULTS"
+ exit ${rc}
+fi
echo ""
echo "Run complete"
diff --git a/docker/scripts/analyze_results.sh
b/docker/scripts/analyze_results.sh
new file mode 100755
index 000..790ec18
--- /dev/null
+++ b/docker/scripts/analyze_results.sh
@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one o
[metron-bro-plugin-kafka] branch master updated: METRON-2017 The Bro plugin docker data processing script incorrectly runs bro (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#30
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new ddfba5c METRON-2017 The Bro plugin docker data processing script
incorrectly runs bro (JonZeolla via ottobackwards) closes
apache/metron-bro-plugin-kafka#30
ddfba5c is described below
commit ddfba5cd37b50984ed2223095a12cc6f9899f34c
Author: JonZeolla
AuthorDate: Thu Feb 28 12:07:18 2019 -0500
METRON-2017 The Bro plugin docker data processing script incorrectly runs
bro (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#30
---
docker/in_docker_scripts/process_data_file.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docker/in_docker_scripts/process_data_file.sh
b/docker/in_docker_scripts/process_data_file.sh
index 81819a2..4ddfdef 100755
--- a/docker/in_docker_scripts/process_data_file.sh
+++ b/docker/in_docker_scripts/process_data_file.sh
@@ -74,7 +74,7 @@ if [ ! -d /root/data ]; then
exit 1
fi
cd /root/test_output/"${OUTPUT_DIRECTORY_NAME}" || exit 1
-find /root/data -type f -name "${PCAP_FILE_NAME}" -print0 | xargs -0 bro -r {}
/usr/local/bro/share/bro/site/local.bro -C
+find /root/data -type f -name "${PCAP_FILE_NAME}" -print0 | xargs -0 bro
/usr/local/bro/share/bro/site/local.bro -C -r
rc=$?; if [[ ${rc} != 0 ]]; then
exit ${rc}
fi
[metron-bro-plugin-kafka] branch master updated: METRON-2004 Bro plugin kafka docker_execute_shell.sh workdir should be unspecified (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git
The following commit(s) were added to refs/heads/master by this push:
new d5f2562 METRON-2004 Bro plugin kafka docker_execute_shell.sh workdir
should be unspecified (JonZeolla via ottobackwards) closes
apache/metron-bro-plugin-kafka#25
d5f2562 is described below
commit d5f2562d26579268ca29dd80c79e0f8af3318a0d
Author: JonZeolla
AuthorDate: Fri Feb 15 09:22:33 2019 -0500
METRON-2004 Bro plugin kafka docker_execute_shell.sh workdir should be
unspecified (JonZeolla via ottobackwards) closes
apache/metron-bro-plugin-kafka#25
---
docker/scripts/docker_execute_shell.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docker/scripts/docker_execute_shell.sh
b/docker/scripts/docker_execute_shell.sh
index ff17a1a..f7c55de 100755
--- a/docker/scripts/docker_execute_shell.sh
+++ b/docker/scripts/docker_execute_shell.sh
@@ -75,5 +75,5 @@ echo "Running build_bro_plugin_docker with "
echo "CONTAINER_NAME = $CONTAINER_NAME"
echo "==="
-docker exec -w /root -i -t "${CONTAINER_NAME}" bash
+docker exec -i -t "${CONTAINER_NAME}" bash
[metron-bro-plugin-kafka] branch master updated: METRON-1911 Create Docker based test environment for Bro Kafka Plugin (ottobackwards) closes apache/metron-bro-plugin-kafka#21
This is an automated email from the ASF dual-hosted git repository. otto pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git The following commit(s) were added to refs/heads/master by this push: new 04593a6 METRON-1911 Create Docker based test environment for Bro Kafka Plugin (ottobackwards) closes apache/metron-bro-plugin-kafka#21 04593a6 is described below commit 04593a6a2d6ee4b3b1d43aab70ad4a406be240fc Author: ottobackwards AuthorDate: Wed Feb 6 09:25:51 2019 -0500 METRON-1911 Create Docker based test environment for Bro Kafka Plugin (ottobackwards) closes apache/metron-bro-plugin-kafka#21 --- .gitignore | 12 + NOTICE | 4 +- docker/README.md | 291 + .../containers/bro-localbuild-container/Dockerfile | 49 docker/data/.gitignore | 4 + docker/finish_end_to_end.sh| 35 +++ docker/in_docker_scripts/build_bro_plugin.sh | 43 +++ docker/in_docker_scripts/configure_bro_plugin.sh | 41 +++ docker/in_docker_scripts/process_data_file.sh | 78 ++ docker/in_docker_scripts/wait-for-it.sh| 162 docker/in_docker_scripts/wait_for_kafka.sh | 35 +++ docker/in_docker_scripts/wait_for_zk.sh| 35 +++ docker/remove_timeout_message.sh | 51 docker/run_end_to_end.sh | 202 ++ docker/scripts/build_container.sh | 111 docker/scripts/cleanup_docker.sh | 91 +++ docker/scripts/create_docker_network.sh| 79 ++ docker/scripts/destroy_docker_network.sh | 76 ++ docker/scripts/docker_execute_build_bro_plugin.sh | 84 ++ .../scripts/docker_execute_configure_bro_plugin.sh | 84 ++ docker/scripts/docker_execute_process_data_file.sh | 113 docker/scripts/docker_execute_shell.sh | 79 ++ docker/scripts/docker_run_bro_container.sh | 168 docker/scripts/docker_run_consume_bro_kafka.sh | 86 ++ .../docker_run_create_bro_topic_in_kafka.sh| 74 ++ docker/scripts/docker_run_get_offset_bro_kafka.sh | 75 ++ docker/scripts/docker_run_kafka_container.sh | 82 ++ docker/scripts/docker_run_wait_for_kafka.sh| 82 ++ docker/scripts/docker_run_wait_for_zookeeper.sh| 82 ++ docker/scripts/docker_run_zookeeper_container.sh | 82 ++ docker/scripts/download_sample_pcaps.sh| 112 docker/scripts/print_results.sh| 91 +++ docker/scripts/split_kakfa_output_by_log.sh| 106 docker/scripts/stop_container.sh | 85 ++ docker/test_output/.gitignore | 4 + 35 files changed, 2887 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 28a8358..735a17e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,9 @@ +#ide stuff +.idea +*.iml +*.iws +.DS_Store + .state build @@ -29,3 +35,9 @@ build *.exe *.out *.app + +# Log files +*.log +# pcap files +*.pcap +*.pcapng diff --git a/NOTICE b/NOTICE index 9505518..7f966de 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,7 @@ Apache Metron - Copyright 2015-2016 The Apache Software Foundation + Copyright 2015-2018 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). + + This product bundles wait-for-it.sh, which is available under a "MIT Software License" license. For details, see https://github.com/vishnubob/wait-for-it diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000..8e4d3fa --- /dev/null +++ b/docker/README.md @@ -0,0 +1,291 @@ + + +## Docker support for testing metron-bro-plugin-kafka + +These scripts and containers provide support for building and testing Bro and the metron-bro-plugin-kafka using a number of Docker containers. +The use of these scripts and containers allow an easier, automated workflow for testing new features, fixes, or regressions than before. +One of the goals is for this to be extensible, such that new scripts can be introduced and run as well. This will allow, for example, one or more +testing scripts to be added to a pull request, and subsequently to a test suite. + + + Directories + +```bash +├── containers +│ └── bro-localbuild-container +├── data +├── in_docker_scripts +├── scripts +└── test_output +``` +- `containers`: The parent of all of the containers that this project defines. We use several containers, not all of them ours. + - `bro-localbuild-container`: The docker container directory for our bro container, used for building bro, the librdkafka, and our plugin, as well as running bro. +- `data`: T
[metron] branch master updated: METRON-1933 Improve build-utils helper scripts (JonZeolla via ottobackwards) closes apache/metron#1297
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 65910d9 METRON-1933 Improve build-utils helper scripts (JonZeolla via
ottobackwards) closes apache/metron#1297
65910d9 is described below
commit 65910d9e7a720ac47a8584510ab75f59c0c9f362
Author: JonZeolla
AuthorDate: Thu Jan 24 07:14:20 2019 -0500
METRON-1933 Improve build-utils helper scripts (JonZeolla via
ottobackwards) closes apache/metron#1297
---
dev-utilities/build-utils/README.md| 3 ++-
dev-utilities/build-utils/list_dependencies.sh | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/dev-utilities/build-utils/README.md
b/dev-utilities/build-utils/README.md
index 495a256..adf5ee7 100644
--- a/dev-utilities/build-utils/README.md
+++ b/dev-utilities/build-utils/README.md
@@ -21,7 +21,8 @@ The aim of the build utilities project is to provide some
scripting
around the care and maintenance of the building infrastructure. At the
moment the primary mission is around utilities that assist us manage the
licenses of our dependencies and generate the appropriate notices or
-licenses.
+licenses. These utilities assume you have already built the repo via `mvn
+clean install -DskipTests`.
## `dependencies_with_url.csv`
This file is the reference file for all of our dependencies.
diff --git a/dev-utilities/build-utils/list_dependencies.sh
b/dev-utilities/build-utils/list_dependencies.sh
index a56a5bd..4e81b7b 100755
--- a/dev-utilities/build-utils/list_dependencies.sh
+++ b/dev-utilities/build-utils/list_dependencies.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -16,4 +16,4 @@
# limitations under the License.
#
-{ mvn dependency:list ; mvn dependency:list -PHDP-2.5.0.0 ; } | grep
"^\[INFO\] " | awk '{print $2}' | grep -v "org.apache" | grep -v "test" |
grep -v "provided" | grep -v "runtime" | grep -v ":system" | sort | uniq
+{ mvn dependency:list || { echo "ERROR: Failed to run mvn dependency:list" ;
exit 1 ; } ; mvn dependency:list -PHDP-2.5.0.0 || { echo "ERROR: Failed to run
mvn dependency:list -PHDP-2.5.0.0" ; exit 1 ; } ; } | grep "^\[INFO\] " | awk
'{print $2}' | grep -v "org.apache" | grep -v "test" | grep -v "provided" |
grep -v "runtime" | grep -v ":system" | sort | uniq
[metron] branch master updated: METRON-1956 prepare-commit does not run all the tests it should (ottobackwards) closes apache/metron#1315
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new d4498e6 METRON-1956 prepare-commit does not run all the tests it
should (ottobackwards) closes apache/metron#1315
d4498e6 is described below
commit d4498e6c512aac37e734fb9e970a3f466632c95c
Author: ottobackwards
AuthorDate: Wed Jan 16 17:17:14 2019 -0500
METRON-1956 prepare-commit does not run all the tests it should
(ottobackwards) closes apache/metron#1315
---
.../committer-utils/metron-committer-common| 198 -
1 file changed, 149 insertions(+), 49 deletions(-)
diff --git a/dev-utilities/committer-utils/metron-committer-common
b/dev-utilities/committer-utils/metron-committer-common
index 30a9bfd..4855254 100644
--- a/dev-utilities/committer-utils/metron-committer-common
+++ b/dev-utilities/committer-utils/metron-committer-common
@@ -51,7 +51,7 @@ CHOSEN_REPO=
function init_configuration {
# does a config file already exist?
echo "$CONFIG_FILE"
-if [ -f ${CONFIG_FILE} ]; then
+if [[ -f ${CONFIG_FILE} ]]; then
#shellcheck source=/dev/null
source ${CONFIG_FILE}
echo " ...using settings from $CONFIG_FILE"
@@ -65,28 +65,28 @@ function init_configuration {
#
function init_committer_info {
# github account of committer (you)
-if [ -z "$GITHUB_NAME" ]; then
+if [[ -z "$GITHUB_NAME" ]]; then
read -p " your github username [$GITHUB_NAME]: " INPUT
-[ -n "$INPUT" ] && GITHUB_NAME=${INPUT}
+[[ -n "$INPUT" ]] && GITHUB_NAME=${INPUT}
# write setting to config file
echo "GITHUB_NAME=$GITHUB_NAME" >> ${CONFIG_FILE}
fi
# apache id of committer (you)
-if [ -z "$APACHE_NAME" ]; then
+if [[ -z "$APACHE_NAME" ]]; then
read -p " your apache userid [$APACHE_NAME]: " INPUT
- [ -n "$INPUT" ] && APACHE_NAME=${INPUT}
+ [[ -n "$INPUT" ]] && APACHE_NAME=${INPUT}
# write setting to config file
echo "APACHE_NAME=$APACHE_NAME" >> ${CONFIG_FILE}
fi
# apache email addr of committer (you)
-if [ -z "$APACHE_EMAIL" ]; then
+if [[ -z "$APACHE_EMAIL" ]]; then
APACHE_EMAIL=${APACHE_NAME}@apache.org
read -p " your apache email [$APACHE_EMAIL]: " INPUT
- [ -n "$INPUT" ] && APACHE_EMAIL=${INPUT}
+ [[ -n "$INPUT" ]] && APACHE_EMAIL=${INPUT}
# write setting to config file, so it is not needed next time
echo "APACHE_EMAIL=$APACHE_EMAIL" >> ${CONFIG_FILE}
@@ -111,7 +111,7 @@ function choose_metron_or_bro_repo {
exit 1
;;
esac
-[ -n "$INPUT" ] && UPSTREAM=${INPUT}
+[[ -n "$INPUT" ]] && UPSTREAM=${INPUT}
CHOSEN_REPO=$(basename ${UPSTREAM%%.git})
}
@@ -122,14 +122,14 @@ function choose_metron_or_bro_repo {
function read_pull_request {
# retrieve the pull request identifier
read -p " pull request: " PR
-if [ -z "$PR" ]; then
+if [[ -z "$PR" ]]; then
echo "Error: missing pr"
exit 1
fi
# ensure that the pull request exists
-PR_EXISTS=`curl -sI
https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/${PR} | grep Status: |
sed 's/[^0-9]//g'`
-if [ "$PR_EXISTS" != "200" ]; then
+PR_EXISTS=$(curl -sI
https://api.github.com/repos/apache/${CHOSEN_REPO}/pulls/${PR} | grep Status: |
sed 's/[^0-9]//g')
+if [[ "$PR_EXISTS" != "200" ]]; then
echo "Error: pull request #$PR does not exist"
exit 1
fi
@@ -142,20 +142,20 @@ function read_pull_request {
#
function setup_working_directory {
# working directory
-if [ -z $1 ]; then
+if [[ -z $1 ]]; then
WORK=~/tmp/${CHOSEN_REPO}-pr${PR}
else
WORK=$1
fi
read -p " local working directory [$WORK]: " INPUT
-[ -n "$INPUT" ] && WORK=${INPUT}
+[[ -n "$INPUT" ]] && WORK=${INPUT}
# handle tilde expansion
WORK="${WORK/#\~/$HOME}"
# warn the user if the working directory exists
-if [ -d "$WORK" ]; then
+if [[ -d "$WORK" ]]; then
read -p " directory exists [$WORK]. continue merge on existing repo?
[yN] " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
@@ -172,27 +172,27 @@ function setup_working_directory {
#
function setup_code {
# if working directory does not exist, checkout the base branch
-if [ ! -d "$WORK&
[metron] branch master updated: METRON-1963 Remove left over integration test from before refactoring (ottobackwards) closes apache/metron#1319
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 8622447 METRON-1963 Remove left over integration test from before
refactoring (ottobackwards) closes apache/metron#1319
8622447 is described below
commit 862244721fef7ba7389000cc2f3d0756bb07d69d
Author: ottobackwards
AuthorDate: Fri Jan 4 12:55:39 2019 -0500
METRON-1963 Remove left over integration test from before refactoring
(ottobackwards) closes apache/metron#1319
---
.../Syslog3164ParserIntegrationTest.java | 37 --
.../parsers/integration/ParserIntegrationTest.java | 1 +
2 files changed, 1 insertion(+), 37 deletions(-)
diff --git
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog3164ParserIntegrationTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog3164ParserIntegrationTest.java
deleted file mode 100644
index e1affe6..000
---
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog3164ParserIntegrationTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.parsers.integration;
-
-import org.apache.metron.parsers.integration.validation.SampleDataValidation;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class Syslog3164ParserIntegrationTest extends ParserIntegrationTest {
- @Override
- String getSensorType() {
-return "syslog3164";
- }
-
- @Override
- List getValidations() {
-return new ArrayList() {{
- add(new SampleDataValidation());
-}};
- }
-}
diff --git
a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
index de0b984..f14e260 100644
---
a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
+++
b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
@@ -44,6 +44,7 @@ public abstract class ParserIntegrationTest extends
BaseIntegrationTest {
"squid",
"websphere",
"yaf",
+ "syslog3164",
"syslog5424"
);
[metron] branch master updated: METRON-1957 5424 and 3164 parser configurations are packaged in wrong place (ottobackwards) closes apache/metron#1316
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 9e717a5 METRON-1957 5424 and 3164 parser configurations are packaged
in wrong place (ottobackwards) closes apache/metron#1316
9e717a5 is described below
commit 9e717a525ddca986130f4208fb6c558eaaa56eab
Author: ottobackwards
AuthorDate: Fri Dec 28 11:02:12 2018 -0500
METRON-1957 5424 and 3164 parser configurations are packaged in wrong place
(ottobackwards) closes apache/metron#1316
---
metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index caf92a5..294e24b 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -157,6 +157,8 @@ This package installs the Metron Parser Common files
%{metron_home}/config/zookeeper/parsers/jsonMap.json
%{metron_home}/config/zookeeper/parsers/jsonMapQuery.json
%{metron_home}/config/zookeeper/parsers/jsonMapWrappedQuery.json
+%{metron_home}/config/zookeeper/parsers/syslog3164.json
+%{metron_home}/config/zookeeper/parsers/syslog5424.json
%{metron_home}/patterns/common
%attr(0644,root,root)
%{metron_home}/lib/metron-parsers-common-%{full_version}-uber.jar
@@ -182,8 +184,6 @@ This package installs the Metron Bundled Parser files
%{metron_home}/config/zookeeper/parsers/bro.json
%{metron_home}/config/zookeeper/parsers/snort.json
%{metron_home}/config/zookeeper/parsers/squid.json
-%{metron_home}/config/zookeeper/parsers/syslog3164.json
-%{metron_home}/config/zookeeper/parsers/syslog5424.json
%{metron_home}/config/zookeeper/parsers/websphere.json
%{metron_home}/config/zookeeper/parsers/yaf.json
%{metron_home}/config/zookeeper/parsers/asa.json
@@ -634,6 +634,8 @@ chkconfig --del metron-management-ui
chkconfig --del metron-alerts-ui
%changelog
+* Thu Dec 27 2018 Apache Metron - 0.7.1
+- Updat metron SPEC to move syslog configurations to right place
* Wed Dec 26 2018 Apache Metron - 0.7.1
- Update metron SPEC file to include syslog 3164 parser
* Thu Nov 15 2018 Apache Metron - 0.7.0
[metron] branch master updated: METRON-1955 Update metron SPEC file to include syslog 3164 parser (anandsubbu via ottobackwards) closes apache/metron#1314
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 7089086 METRON-1955 Update metron SPEC file to include syslog 3164
parser (anandsubbu via ottobackwards) closes apache/metron#1314
7089086 is described below
commit 7089086cfc8c070a48d80b3884ad396ebb025238
Author: anandsubbu
AuthorDate: Wed Dec 26 11:10:38 2018 -0500
METRON-1955 Update metron SPEC file to include syslog 3164 parser
(anandsubbu via ottobackwards) closes apache/metron#1314
---
metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec | 3 +++
1 file changed, 3 insertions(+)
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index a16a863..caf92a5 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -182,6 +182,7 @@ This package installs the Metron Bundled Parser files
%{metron_home}/config/zookeeper/parsers/bro.json
%{metron_home}/config/zookeeper/parsers/snort.json
%{metron_home}/config/zookeeper/parsers/squid.json
+%{metron_home}/config/zookeeper/parsers/syslog3164.json
%{metron_home}/config/zookeeper/parsers/syslog5424.json
%{metron_home}/config/zookeeper/parsers/websphere.json
%{metron_home}/config/zookeeper/parsers/yaf.json
@@ -633,6 +634,8 @@ chkconfig --del metron-management-ui
chkconfig --del metron-alerts-ui
%changelog
+* Wed Dec 26 2018 Apache Metron - 0.7.1
+- Update metron SPEC file to include syslog 3164 parser
* Thu Nov 15 2018 Apache Metron - 0.7.0
- Split metron-parsers into metron-parsing and submodules
* Wed Oct 31 2018 Apache Metron - 0.7.0
[metron] branch master updated: METRON-1893 Create a syslog 3164 parser (ottobackwards) closes apache/metron#1279
This is an automated email from the ASF dual-hosted git repository.
otto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 0dc9fc8 METRON-1893 Create a syslog 3164 parser (ottobackwards)
closes apache/metron#1279
0dc9fc8 is described below
commit 0dc9fc8fe862c6a43f25bf9c61bbd38a36d3bbc8
Author: ottobackwards
AuthorDate: Mon Dec 24 11:00:39 2018 -0500
METRON-1893 Create a syslog 3164 parser (ottobackwards) closes
apache/metron#1279
---
dependencies_with_url.csv | 2 +-
.../sample/data/syslog3164/parsed/Syslog3164Parsed | 100 +++
.../sample/data/syslog3164/raw/Syslog3164Output| 100 +++
.../sample/data/syslog5424/parsed/Syslog5424Parsed | 6 +-
.../Syslog3164ParserIntegrationTest.java | 37
metron-platform/metron-parsing/README.md | 1 +
.../metron-parsing/metron-parsers-common/README.md | 1 +
.../metron-parsing/metron-parsers-common/pom.xml | 5 +
.../main/config/zookeeper/parsers/syslog3164.json | 6 +
.../main/config/zookeeper/parsers/syslog5424.json | 0
.../metron/parsers/syslog/BaseSyslogParser.java} | 89 ++
.../metron/parsers/syslog/Syslog3164Parser.java| 43 +
.../metron/parsers/syslog/Syslog5424Parser.java| 51 ++
.../parsers/syslog/Syslog3164ParserTest.java | 187 +
.../parsers/syslog/Syslog5424ParserTest.java | 49 +-
.../metron-parsing/metron-parsers/README.md| 1 -
.../metron-parsing/metron-parsers/pom.xml | 5 -
.../src/main/resources/META-INF/NOTICE | 6 +
pom.xml| 2 +-
19 files changed, 641 insertions(+), 50 deletions(-)
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 17453f5..745e3c9 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -488,7 +488,7 @@ org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile
org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
com.zaxxer:HikariCP:jar:2.7.8:compile,ASLv2,https://github.com/brettwooldridge/HikariCP
org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https://github.com/hibernate/hibernate-validator
-com.github.palindromicity:simple-syslog-5424:jar:0.0.9:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
+com.github.palindromicity:simple-syslog:jar:0.0.1:compile,ASLv2,https://github.com/palindromicity/simple-syslog
org.elasticsearch.client:elasticsearch-rest-high-level-client:jar:5.6.14:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
org.elasticsearch.plugin:aggs-matrix-stats-client:jar:5.6.14:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
org.fusesource.jansi:jansi:jar:1.16:compile,ASLv2,https://github.com/fusesource/jansi/blob/master/license.txt
diff --git
a/metron-platform/metron-integration-test/src/main/sample/data/syslog3164/parsed/Syslog3164Parsed
b/metron-platform/metron-integration-test/src/main/sample/data/syslog3164/parsed/Syslog3164Parsed
new file mode 100644
index 000..4e90b46
--- /dev/null
+++
b/metron-platform/metron-integration-test/src/main/sample/data/syslog3164/parsed/Syslog3164Parsed
@@ -0,0 +1,100 @@
+{"syslog.header.hostName":"10.22.8.216","original_string":"<167>Jan 5
08:52:35 10.22.8.216 %ASA-7-609001: Built local-host
inside:10.22.8.205","syslog.header.facility":"20","guid":"4f2beee4-c6d3-4282-b5e1-be42417e717e","syslog.header.timestamp":"Jan
5 08:52:35","syslog.message":"%ASA-7-609001: Built local-host
inside:10.22.8.205","syslog.header.pri":"167","syslog.header.severity":"7","timestamp":1515142355000,"source.type":"syslog3164"}
+{"syslog.header.hostName":"10.22.8.216","original_string":"<166>Jan 5
08:52:35 10.22.8.216 %ASA-6-302021: Teardown ICMP connection for faddr
10.22.8.74\/0(LOCAL\\user.name) gaddr 10.22.8.205\/0 laddr
10.22.8.205\/0","syslog.header.facility":"20","guid":"4e86e51e-a970-4a96-bb79-7d400030755c","syslog.header.timestamp":"Jan
5 08:52:35","syslog.message":"%ASA-6-302021: Teardown ICMP connection for
faddr 10.22.8.74\/0(LOCAL\\user.name) gaddr 10.22.8.205\/0 laddr
10.22.8.205\/0","syslog.head [...]
+{"syslog.header.hostName":"10.22.8.216","original_string":"<167>Jan 5
08:52:35 10.22.8.216 %ASA-7-609002: Teardown local-host inside:10.22.8.205
duration
0:00:00","syslog.header.facility":"20","guid":"430bbc53-48e9-4f57-bfa6-18a28b7b0223","
metron git commit: METRON-1890 Metron Vagrant should disable audio (ottobackwards) closes apache/metron#1277
Repository: metron
Updated Branches:
refs/heads/master d599efb08 -> 12e3ae548
METRON-1890 Metron Vagrant should disable audio (ottobackwards) closes
apache/metron#1277
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/12e3ae54
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/12e3ae54
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/12e3ae54
Branch: refs/heads/master
Commit: 12e3ae548af6bd408a31fde0d1d8cf23cefdf325
Parents: d599efb
Author: ottobackwards
Authored: Wed Nov 21 09:28:01 2018 -0500
Committer: otto
Committed: Wed Nov 21 09:28:01 2018 -0500
--
metron-deployment/development/centos6/Vagrantfile | 2 ++
metron-deployment/development/fastcapa/centos-7.1/Vagrantfile | 2 ++
metron-deployment/development/fastcapa/centos-7.4/Vagrantfile | 2 ++
metron-deployment/development/ubuntu14/Vagrantfile| 2 ++
4 files changed, 8 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/12e3ae54/metron-deployment/development/centos6/Vagrantfile
--
diff --git a/metron-deployment/development/centos6/Vagrantfile
b/metron-deployment/development/centos6/Vagrantfile
index 05823a2..50571b4 100644
--- a/metron-deployment/development/centos6/Vagrantfile
+++ b/metron-deployment/development/centos6/Vagrantfile
@@ -88,6 +88,8 @@ Vagrant.configure(2) do |config|
if host.has_key?(:promisc)
vb.customize ["modifyvm", :id, "--nicpromisc#{host[:promisc]}",
"allow-all"]
end
+# disable audio, so that the vm doesn't capture the sound / mic
+vb.customize ["modifyvm", :id, "--audio", "none"]
end
end
end
http://git-wip-us.apache.org/repos/asf/metron/blob/12e3ae54/metron-deployment/development/fastcapa/centos-7.1/Vagrantfile
--
diff --git a/metron-deployment/development/fastcapa/centos-7.1/Vagrantfile
b/metron-deployment/development/fastcapa/centos-7.1/Vagrantfile
index 179ca34..39c8d32 100644
--- a/metron-deployment/development/fastcapa/centos-7.1/Vagrantfile
+++ b/metron-deployment/development/fastcapa/centos-7.1/Vagrantfile
@@ -58,6 +58,8 @@ Vagrant.configure("2") do |config|
# network adapter settings;
[Am79C970A|Am79C973|82540EM|82543GC|82545EM|virtio]
vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
vb.customize ["modifyvm", :id, "--nictype2","82545EM"]
+ # disable audio, so that the vm doesn't capture the sound / mic
+ vb.customize ["modifyvm", :id, "--audio", "none"]
end
# provision host
http://git-wip-us.apache.org/repos/asf/metron/blob/12e3ae54/metron-deployment/development/fastcapa/centos-7.4/Vagrantfile
--
diff --git a/metron-deployment/development/fastcapa/centos-7.4/Vagrantfile
b/metron-deployment/development/fastcapa/centos-7.4/Vagrantfile
index c62a0f1..3e55d09 100644
--- a/metron-deployment/development/fastcapa/centos-7.4/Vagrantfile
+++ b/metron-deployment/development/fastcapa/centos-7.4/Vagrantfile
@@ -58,6 +58,8 @@ Vagrant.configure("2") do |config|
# network adapter settings;
[Am79C970A|Am79C973|82540EM|82543GC|82545EM|virtio]
vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
vb.customize ["modifyvm", :id, "--nictype2","82545EM"]
+ # disable audio, so that the vm doesn't capture the sound / mic
+ vb.customize ["modifyvm", :id, "--audio", "none"]
end
# provision host
http://git-wip-us.apache.org/repos/asf/metron/blob/12e3ae54/metron-deployment/development/ubuntu14/Vagrantfile
--
diff --git a/metron-deployment/development/ubuntu14/Vagrantfile
b/metron-deployment/development/ubuntu14/Vagrantfile
index facc3cc..ab44491 100644
--- a/metron-deployment/development/ubuntu14/Vagrantfile
+++ b/metron-deployment/development/ubuntu14/Vagrantfile
@@ -83,6 +83,8 @@ Vagrant.configure(2) do |config|
node.vm.provider "virtualbox" do |vb|
vb.memory = host[:memory]
vb.cpus = host[:cpus]
+# disable audio, so that the vm doesn't capture the sound / mic
+vb.customize ["modifyvm", :id, "--audio", "none"]
end
end
end
[46/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-alerts/cypress/fixtures/pcap.status-00.json
--
diff --git
a/metron-interface/metron-alerts/cypress/fixtures/pcap.status-00.json
b/metron-interface/metron-alerts/cypress/fixtures/pcap.status-00.json
new file mode 100644
index 000..2a3cdcc
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress/fixtures/pcap.status-00.json
@@ -0,0 +1,7 @@
+{
+ "jobId":"job_1537878471649_0001",
+ "jobStatus":"RUNNING",
+ "description":"map: 0.0%, reduce: 0.0%",
+ "percentComplete":0.0,
+ "pageTotal":0
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-alerts/cypress/fixtures/pcap.status-01.json
--
diff --git
a/metron-interface/metron-alerts/cypress/fixtures/pcap.status-01.json
b/metron-interface/metron-alerts/cypress/fixtures/pcap.status-01.json
new file mode 100644
index 000..1505f71
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress/fixtures/pcap.status-01.json
@@ -0,0 +1,7 @@
+{
+ "jobId":"job_1537878471649_0001",
+ "jobStatus":"RUNNING",
+ "description":"map: 100.0%, reduce: 100.0%",
+ "percentComplete":75.0,
+ "pageTotal":0
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-alerts/cypress/fixtures/pcap.status-02.json
--
diff --git
a/metron-interface/metron-alerts/cypress/fixtures/pcap.status-02.json
b/metron-interface/metron-alerts/cypress/fixtures/pcap.status-02.json
new file mode 100644
index 000..662c27a
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress/fixtures/pcap.status-02.json
@@ -0,0 +1,7 @@
+{
+ "jobId":"job_1537878471649_0001",
+ "jobStatus":"SUCCEEDED",
+ "description":"Job completed.",
+ "percentComplete":100.0,
+ "pageTotal":2
+}
\ No newline at end of file
[37/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/app/sensors/sensor-storm-settings/sensor-storm-settings.component.spec.ts
--
diff --git
a/metron-interface/metron-config/src/app/sensors/sensor-storm-settings/sensor-storm-settings.component.spec.ts
b/metron-interface/metron-config/src/app/sensors/sensor-storm-settings/sensor-storm-settings.component.spec.ts
index 02f1fd9..6549b4b 100644
---
a/metron-interface/metron-config/src/app/sensors/sensor-storm-settings/sensor-storm-settings.component.spec.ts
+++
b/metron-interface/metron-config/src/app/sensors/sensor-storm-settings/sensor-storm-settings.component.spec.ts
@@ -16,153 +16,161 @@
* limitations under the License.
*/
-import {async, TestBed, ComponentFixture} from '@angular/core/testing';
-import {SensorStormSettingsComponent} from './sensor-storm-settings.component';
-import {SharedModule} from '../../shared/shared.module';
-import {SimpleChanges, SimpleChange} from '@angular/core';
-import {SensorParserConfig} from '../../model/sensor-parser-config';
-import {SensorStormSettingsModule} from './sensor-storm-settings.module';
-import '../../rxjs-operators';
+import { async, TestBed, ComponentFixture } from '@angular/core/testing';
+import { SensorStormSettingsComponent } from
'./sensor-storm-settings.component';
+import { SharedModule } from '../../shared/shared.module';
+import { SimpleChanges, SimpleChange } from '@angular/core';
+import { SensorParserConfig } from '../../model/sensor-parser-config';
+import { SensorStormSettingsModule } from './sensor-storm-settings.module';
describe('Component: SensorStormSettingsComponent', () => {
-
-let fixture: ComponentFixture;
-let component: SensorStormSettingsComponent;
-let sensorParserConfig: SensorParserConfig = new SensorParserConfig();
-sensorParserConfig.sensorTopic = 'bro';
-sensorParserConfig.parserClassName =
'org.apache.metron.parsers.bro.BasicBroParser';
-sensorParserConfig.parserConfig = {};
-sensorParserConfig.numWorkers = 2;
-sensorParserConfig.numAckers = 2;
-sensorParserConfig.spoutParallelism = 2;
-sensorParserConfig.spoutNumTasks = 2;
-sensorParserConfig.parserParallelism = 2;
-sensorParserConfig.parserNumTasks = 2;
-sensorParserConfig.errorWriterParallelism = 2;
-sensorParserConfig.errorWriterNumTasks = 2;
-sensorParserConfig.spoutConfig = {'spoutConfigProp': 'spoutConfigValue1'};
-sensorParserConfig.stormConfig = {'stormConfigProp': 'stormConfigValue1'};
-
-beforeEach(async(() => {
-TestBed.configureTestingModule({
-imports: [SharedModule, SensorStormSettingsModule],
-});
-
-fixture = TestBed.createComponent(SensorStormSettingsComponent);
-component = fixture.componentInstance;
-}));
-
-it('should create an instance', () => {
-expect(component).toBeDefined();
-});
-
-it('should create an instance', () => {
-spyOn(component, 'init');
-let changes: SimpleChanges = {'showStormSettings': new
SimpleChange(false, true)};
-
-component.ngOnChanges(changes);
-expect(component.init).toHaveBeenCalled();
-
-changes = {'showStormSettings': new SimpleChange(true, false)};
-component.ngOnChanges(changes);
-expect(component.init['calls'].count()).toEqual(1);
-
-fixture.destroy();
+ let fixture: ComponentFixture;
+ let component: SensorStormSettingsComponent;
+ let sensorParserConfig: SensorParserConfig = new SensorParserConfig();
+ sensorParserConfig.sensorTopic = 'bro';
+ sensorParserConfig.parserClassName =
+'org.apache.metron.parsers.bro.BasicBroParser';
+ sensorParserConfig.parserConfig = {};
+ sensorParserConfig.numWorkers = 2;
+ sensorParserConfig.numAckers = 2;
+ sensorParserConfig.spoutParallelism = 2;
+ sensorParserConfig.spoutNumTasks = 2;
+ sensorParserConfig.parserParallelism = 2;
+ sensorParserConfig.parserNumTasks = 2;
+ sensorParserConfig.errorWriterParallelism = 2;
+ sensorParserConfig.errorWriterNumTasks = 2;
+ sensorParserConfig.spoutConfig = { spoutConfigProp: 'spoutConfigValue1' };
+ sensorParserConfig.stormConfig = { stormConfigProp: 'stormConfigValue1' };
+
+ beforeEach(async(() => {
+TestBed.configureTestingModule({
+ imports: [SharedModule, SensorStormSettingsModule]
});
-it('should initialise the fields', () => {
-
-component.init();
-expect(component.newSensorParserConfig).toEqual(new
SensorParserConfig());
-
-component.sensorParserConfig = sensorParserConfig;
-component.init();
-expect(component.newSensorParserConfig).toEqual(sensorParserConfig);
-expect(component.newSpoutConfig).toEqual('{\n\t"spoutConfigProp":
"spoutConfigValue1"\n}');
-expect(component.newStormConfig).toEqual('{\n\t"stormConfigProp":
"stormConfigValue1"\n}');
-
-
[40/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/app/sensors/sensor-field-schema/sensor-field-schema.component.spec.ts -- diff --git a/metron-interface/metron-config/src/app/sensors/sensor-field-schema/sensor-field-schema.component.spec.ts b/metron-interface/metron-config/src/app/sensors/sensor-field-schema/sensor-field-schema.component.spec.ts index d2066ea..2cb44e2 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-field-schema/sensor-field-schema.component.spec.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-field-schema/sensor-field-schema.component.spec.ts @@ -19,505 +19,641 @@ /* tslint:disable:max-line-length */ import { TestBed, async, ComponentFixture } from '@angular/core/testing'; -import {Http} from '@angular/http'; -import {SimpleChanges, SimpleChange} from '@angular/core'; -import {SensorParserConfigService} from '../../service/sensor-parser-config.service'; -import {StellarService} from '../../service/stellar.service'; -import {MetronAlerts} from '../../shared/metron-alerts'; -import {SensorFieldSchemaModule} from './sensor-field-schema.module'; -import {SensorFieldSchemaComponent, FieldSchemaRow} from './sensor-field-schema.component'; -import {KafkaService} from '../../service/kafka.service'; -import {Observable} from 'rxjs/Observable'; -import {StellarFunctionDescription} from '../../model/stellar-function-description'; -import {SensorParserConfig} from '../../model/sensor-parser-config'; -import {SensorEnrichmentConfig, EnrichmentConfig, ThreatIntelConfig} from '../../model/sensor-enrichment-config'; -import {ParseMessageRequest} from '../../model/parse-message-request'; -import {AutocompleteOption} from '../../model/autocomplete-option'; -import {FieldTransformer} from '../../model/field-transformer'; -import {SensorEnrichmentConfigService} from '../../service/sensor-enrichment-config.service'; - +import { HttpClient } from '@angular/common/http'; +import { SimpleChanges, SimpleChange } from '@angular/core'; +import { SensorParserConfigService } from '../../service/sensor-parser-config.service'; +import { StellarService } from '../../service/stellar.service'; +import { MetronAlerts } from '../../shared/metron-alerts'; +import { SensorFieldSchemaModule } from './sensor-field-schema.module'; +import { + SensorFieldSchemaComponent, + FieldSchemaRow +} from './sensor-field-schema.component'; +import { KafkaService } from '../../service/kafka.service'; +import { Observable, throwError } from 'rxjs'; +import { StellarFunctionDescription } from '../../model/stellar-function-description'; +import { SensorParserConfig } from '../../model/sensor-parser-config'; +import { + SensorEnrichmentConfig, + EnrichmentConfig, + ThreatIntelConfig +} from '../../model/sensor-enrichment-config'; +import { ParseMessageRequest } from '../../model/parse-message-request'; +import { AutocompleteOption } from '../../model/autocomplete-option'; +import { FieldTransformer } from '../../model/field-transformer'; +import { SensorEnrichmentConfigService } from '../../service/sensor-enrichment-config.service'; class MockSensorParserConfigService { - -parseMessage(parseMessageRequest: ParseMessageRequest): Observable<{}> { -let parsedJson = { -'elapsed': 415, -'code': 200, -'ip_dst_addr': '207.109.73.154', -'original_string': '1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/', -'method': 'GET', -'bytes': 337891, -'action': 'TCP_MISS', -'ip_src_addr': '127.0.0.1', -'url': 'http://www.aliexpress.com/af/shoes.html?', -'timestamp': '1467011157.401' -}; -return Observable.create((observable) => { -observable.next(parsedJson); -observable.complete(); -}); -} + parseMessage(parseMessageRequest: ParseMessageRequest): Observable<{}> { +let parsedJson = { + elapsed: 415, + code: 200, + ip_dst_addr: '207.109.73.154', + original_string: +'1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/', + method: 'GET', + bytes: 337891, + action: 'TCP_MISS', + ip_src_addr: '127.0.0.1', + url: 'http://www.aliexpress.com/af/shoes.html?', + timestamp: '1467011157.401' +}; +return Observable.create(observable => { +
[42/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/package-lock.json
--
diff --git a/metron-interface/metron-config/package-lock.json
b/metron-interface/metron-config/package-lock.json
index fff8cb7..f2a264d 100644
--- a/metron-interface/metron-config/package-lock.json
+++ b/metron-interface/metron-config/package-lock.json
@@ -4,180 +4,530 @@
"lockfileVersion": 1,
"requires": true,
"dependencies": {
-"@angular-cli/ast-tools": {
- "version": "1.0.16",
- "resolved":
"https://registry.npmjs.org/@angular-cli/ast-tools/-/ast-tools-1.0.16.tgz;,
- "integrity": "sha1-YxmULBol+4TjKUID6fejJmMvzlA=",
+"@angular-devkit/architect": {
+ "version": "0.8.3",
+ "resolved":
"https://registry.npmjs.org/@angular-devkit/architect/-/architect-0.8.3.tgz;,
+ "integrity":
"sha512-cFku50grgEJPg1CZZ0DXt4CkA6WnV6zN3hCXzpWbOfc/Id923Mml/jsEaoByeXHsRqb5rIZKZAhz7R509ya8OQ==",
"dev": true,
"requires": {
-"@angular/tsc-wrapped": "0.5.2",
-"denodeify": "1.2.1",
-"rxjs": "5.1.0",
-"typescript": "2.0.10"
+"@angular-devkit/core": "0.8.3",
+"rxjs": "6.2.2"
},
"dependencies": {
-"@angular/tsc-wrapped": {
- "version": "0.5.2",
- "resolved":
"https://registry.npmjs.org/@angular/tsc-wrapped/-/tsc-wrapped-0.5.2.tgz;,
- "integrity": "sha1-Lt30csRn/LM06pTe3aqnGZDFpII=",
+"@angular-devkit/core": {
+ "version": "0.8.3",
+ "resolved":
"https://registry.npmjs.org/@angular-devkit/core/-/core-0.8.3.tgz;,
+ "integrity":
"sha512-2KHt5osMs3zACYXev20ZU5SXdWoinoKwZkj2caj2LCj9W7QNHmsz34QvaygNq7YdJzF3jkXkdy0GSUgUgDke0w==",
"dev": true,
"requires": {
-"tsickle": "0.2.5"
+"ajv": "6.4.0",
+"chokidar": "2.0.4",
+"rxjs": "6.2.2",
+"source-map": "0.5.7"
}
-},
-"denodeify": {
- "version": "1.2.1",
- "resolved":
"https://registry.npmjs.org/denodeify/-/denodeify-1.2.1.tgz;,
- "integrity": "sha1-OjYof1A05pnnV3kBBSwubJQlFjE=",
+}
+ }
+},
+"@angular-devkit/build-angular": {
+ "version": "0.8.3",
+ "resolved":
"https://registry.npmjs.org/@angular-devkit/build-angular/-/build-angular-0.8.3.tgz;,
+ "integrity":
"sha512-NWwWV+6apvCGmllWjwwy9Pmj5uK5tVGL/xIVQgSGC5waLmW/vFWNRXCI50ji5UPP+vAeRi/pWdXWMxuoVA08FA==",
+ "dev": true,
+ "requires": {
+"@angular-devkit/architect": "0.8.3",
+"@angular-devkit/build-optimizer": "0.8.3",
+"@angular-devkit/build-webpack": "0.8.3",
+"@angular-devkit/core": "0.8.3",
+"@ngtools/webpack": "6.2.3",
+"ajv": "6.4.0",
+"autoprefixer": "8.6.5",
+"circular-dependency-plugin": "5.0.2",
+"clean-css": "4.2.1",
+"copy-webpack-plugin": "4.5.2",
+"file-loader": "1.1.11",
+"glob": "7.1.3",
+"html-webpack-plugin": "3.2.0",
+"istanbul": "0.4.5",
+"istanbul-instrumenter-loader": "3.0.1",
+"karma-source-map-support": "1.3.0",
+"less": "3.8.1",
+"less-loader": "4.1.0",
+"license-webpack-plugin": "1.5.0",
+"loader-utils": "1.1.0",
+"mini-css-extract-plugin": "0.4.3",
+"minimatch": "3.0.4",
+"node-sass": "4.9.3",
+"opn": "5.3.0",
+"parse5": "4.0.0",
+"portfinder": "1.0.17",
+"postcss": "6.0.23",
+"postcss-import": "11.1.0",
+"postcss-loader": "2.1.6",
+"postcss-url": "7.3.2",
+"raw-loader": "0.5.1",
+"rxjs": "6.2.2",
+"sass-loader": "7.1.0",
+"semver": "5.5.1",
+"source-map-loader": "0.2.4",
+"source-map-support": "0.5.9",
+"stats-webpack-plugin": "0.6.2",
+"style-loader": "0.21.0",
+"stylus": "0.54.5",
+"stylus-loader": "3.0.2",
+"tree-kill": "1.2.0",
+"uglifyjs-webpack-plugin": "1.3.0",
+"url-loader": "1.1.1",
+"webpack": "4.19.1",
+"webpack-dev-middleware": "3.3.0",
+"webpack-dev-server": "3.1.8",
+"webpack-merge": "4.1.4",
+"webpack-sources": "1.3.0",
+"webpack-subresource-integrity": "1.1.0-rc.6"
+ },
+ "dependencies": {
+"@angular-devkit/core": {
+ "version": "0.8.3",
+ "resolved":
"https://registry.npmjs.org/@angular-devkit/core/-/core-0.8.3.tgz;,
+ "integrity":
"sha512-2KHt5osMs3zACYXev20ZU5SXdWoinoKwZkj2caj2LCj9W7QNHmsz34QvaygNq7YdJzF3jkXkdy0GSUgUgDke0w==",
+ "dev": true,
+ "requires": {
+"ajv": "6.4.0",
+"chokidar": "2.0.4",
+"rxjs": "6.2.2",
+"source-map": "0.5.7"
+ }
+}
+ }
+},
+"@angular-devkit/build-optimizer": {
+ "version": "0.8.3",
+
[17/52] [abbrv] metron git commit: METRON-1870 Intermittent Stellar REST test failures (merrimanr via nickwallen) closes apache/metron#1263
METRON-1870 Intermittent Stellar REST test failures (merrimanr via nickwallen)
closes apache/metron#1263
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0e2a4348
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0e2a4348
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0e2a4348
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 0e2a4348465406ad3aca7614bc38a71571b63b83
Parents: 0303d24
Author: merrimanr
Authored: Wed Nov 14 13:04:11 2018 -0500
Committer: nickallen
Committed: Wed Nov 14 13:04:11 2018 -0500
--
.../stellar/dsl/functions/RestFunctions.java | 4 ++--
.../stellar/dsl/functions/RestFunctionsTest.java | 17 +++--
2 files changed, 9 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/0e2a4348/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
index 7134bfc..354322a 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
@@ -223,14 +223,14 @@ public class RestFunctions {
* @return
* @throws IOException
*/
-private Object doGet(RestConfig restConfig, HttpGet httpGet,
HttpClientContext httpClientContext) throws IOException {
+protected Object doGet(RestConfig restConfig, HttpGet httpGet,
HttpClientContext httpClientContext) throws IOException {
// Schedule a command to abort the httpGet request if the timeout is
exceeded
ScheduledFuture scheduledFuture =
scheduledExecutorService.schedule(httpGet::abort, restConfig.getTimeout(),
TimeUnit.MILLISECONDS);
CloseableHttpResponse response;
try {
response = httpClient.execute(httpGet, httpClientContext);
- } catch(IOException e) {
+ } catch(Exception e) {
// Report a timeout if the httpGet request was aborted. Otherwise
rethrow exception.
if (httpGet.isAborted()) {
throw new IOException(String.format("Total Stellar REST request time
to %s exceeded the configured timeout of %d ms.", httpGet.getURI().toString(),
restConfig.getTimeout()));
http://git-wip-us.apache.org/repos/asf/metron/blob/0e2a4348/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
--
diff --git
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
index ba80f02..4b912ef 100644
---
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
+++
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
@@ -23,10 +23,9 @@ import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
-import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
@@ -66,7 +65,10 @@ import static
org.apache.metron.stellar.dsl.functions.RestConfig.STELLAR_REST_SE
import static org.apache.metron.stellar.dsl.functions.RestConfig.TIMEOUT;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
@@ -552,15 +554,10 @@ public class RestFunctionsTest {
*/
@Test
public void restGetShouldHandleIOException() throws
IllegalArgumentException, IOException {
-Map globalConfig = new HashMap() {{
- put(STELLAR_REST_SETTINGS, new HashMap() {{
-put(SOCKET_TIMEOUT, 1);
-
[14/52] [abbrv] metron git commit: METRON-1847 Create reusable script with functions from prepare-commit (ottobackwards) closes apache/metron#1248
METRON-1847 Create reusable script with functions from prepare-commit
(ottobackwards) closes apache/metron#1248
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/42068d95
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/42068d95
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/42068d95
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 42068d95192e7346bf0179399193a6afe01d9925
Parents: 3e73391
Author: ottobackwards
Authored: Thu Nov 8 22:14:38 2018 -0500
Committer: otto
Committed: Thu Nov 8 22:14:38 2018 -0500
--
.../committer-utils/metron-committer-common | 357 +++
dev-utilities/committer-utils/prepare-commit| 241 +
2 files changed, 372 insertions(+), 226 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/42068d95/dev-utilities/committer-utils/metron-committer-common
--
diff --git a/dev-utilities/committer-utils/metron-committer-common
b/dev-utilities/committer-utils/metron-committer-common
new file mode 100644
index 000..22e19a2
--- /dev/null
+++ b/dev-utilities/committer-utils/metron-committer-common
@@ -0,0 +1,357 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# common and unlikely to change vars
+#
+
+# the upstream apache git repo for apache metron
+METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron.git;
+# the upstream apache git repo for apache metron bro plugin kafka
+BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git;
+# the common configuration file with the committer info
+CONFIG_FILE=~/.metron-prepare-commit
+
+GITHUB_REMOTE="origin"
+BASE_BRANCH=master
+
+# other var setup by these functions
+PR=
+WORK=
+ORIGIN=
+UPSTREAM=
+PR_BRANCH=
+USER=
+EMAIL=
+JIRA=
+DESC=
+APACHE_NAME=
+APACHE_EMAIL=
+GITHUB_NAME=
+
+#
+# Initialize the variables from the default configuration file, if it exists
+#
+function init_configuration {
+# does a config file already exist?
+echo "$CONFIG_FILE"
+if [ -f ${CONFIG_FILE} ]; then
+#shellcheck source=/dev/null
+source ${CONFIG_FILE}
+echo " ...using settings from $CONFIG_FILE"
+fi
+}
+
+
+#
+# Initialize the committer variables if they are not provided through the
configuration file.
+# If it is not present, it will be written out for the next time
+#
+function init_committer_info {
+# github account of committer (you)
+if [ -z "$GITHUB_NAME" ]; then
+read -p " your github username [$GITHUB_NAME]: " INPUT
+[ -n "$INPUT" ] && GITHUB_NAME=${INPUT}
+
+# write setting to config file
+echo "GITHUB_NAME=$GITHUB_NAME" >> ${CONFIG_FILE}
+fi
+
+# apache id of committer (you)
+if [ -z "$APACHE_NAME" ]; then
+ read -p " your apache userid [$APACHE_NAME]: " INPUT
+ [ -n "$INPUT" ] && APACHE_NAME=${INPUT}
+
+ # write setting to config file
+ echo "APACHE_NAME=$APACHE_NAME" >> ${CONFIG_FILE}
+fi
+
+# apache email addr of committer (you)
+if [ -z "$APACHE_EMAIL" ]; then
+ APACHE_EMAIL=${APACHE_NAME}@apache.org
+ read -p " your apache email [$APACHE_EMAIL]: " INPUT
+ [ -n "$INPUT" ] && APACHE_EMAIL=${INPUT}
+
+ # write setting to config file, so it is not needed next time
+ echo "APACHE_EMAIL=$APACHE_EMAIL" >> ${CONFIG_FILE}
+fi
+}
+
+#
+# Provide the user with a choice of the metron or bro repositories
+#
+function choose_metron_or_bro_repo {
+# which repo? metron or metron-bro-plugin-kafka
+echo "[1] metron"
+echo "[2] metron-bro-plugin-kafka"
+read -p " which repo? [1]: " INPUT
+case "${INPUT}" in
+
[Bb][Rr][Oo]|[Mm][Ee][Tt][Rr][Oo]
[49/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8bf3b6ec Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8bf3b6ec Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8bf3b6ec Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 8bf3b6ec9c6871daadcaa2341d01082d2584e341 Parents: e7e19fb Author: mmiklavc Authored: Thu Nov 15 14:16:15 2018 -0700 Committer: Michael Miklavcic Committed: Thu Nov 15 16:51:22 2018 -0700 -- .gitignore | 1 + Upgrading.md|24 + dependencies_with_url.csv | 5 +- .../committer-utils/metron-committer-common | 358 + dev-utilities/committer-utils/prepare-commit| 241 +- dev-utilities/release-utils/README.md | 129 + dev-utilities/release-utils/metron-rc-check | 6 +- .../release-utils/prepare-release-candidate | 334 + metron-analytics/metron-maas-common/pom.xml | 2 +- .../metron-profiler-spark/README.md |53 +- .../metron/profiler/spark/BatchProfiler.java|21 +- .../profiler/spark/BatchProfilerConfig.java | 9 +- .../reader/ColumnEncodedTelemetryReader.java|84 + .../profiler/spark/reader/TelemetryReader.java |43 + .../profiler/spark/reader/TelemetryReaders.java | 110 + .../reader/TextEncodedTelemetryReader.java |83 + .../spark/BatchProfilerIntegrationTest.java |36 +- .../ColumnEncodedTelemetryReaderTest.java | 118 + .../spark/function/reader/IsValidJSON.java |38 + .../function/reader/TelemetryReadersTest.java |89 + .../reader/TextEncodedTelemetryReaderTest.java | 114 + metron-deployment/Kerberos-manual-setup.md | 154 +- .../roles/ambari_master/defaults/main.yml | 1 + .../ambari_master/tasks/elasticsearch_mpack.yml | 4 +- .../ansible/roles/bro/tasks/bro.yml | 4 +- .../ansible/roles/bro/tasks/dependencies.yml| 4 +- .../roles/bro/tasks/metron-bro-plugin-kafka.yml | 4 +- .../ansible/roles/librdkafka/defaults/main.yml | 4 +- metron-deployment/development/README.md |34 + metron-deployment/development/centos6/README.md |26 +- .../development/knox-demo-ldap.ldif | 101 + .../development/ubuntu14/README.md |26 +- .../configuration/metron-enrichment-env.xml | 8 +- .../configuration/metron-indexing-env.xml | 4 +- .../configuration/metron-profiler-env.xml | 2 +- .../CURRENT/configuration/metron-rest-env.xml |10 +- .../configuration/metron-security-env.xml | 186 + .../common-services/METRON/CURRENT/metainfo.xml | 3 + .../CURRENT/package/files/bro_index.template| 3 + .../package/files/metaalert_index.template | 4 +- .../CURRENT/package/files/snort_index.template | 3 + .../CURRENT/package/files/yaf_index.template| 3 + .../package/scripts/params/params_linux.py |27 +- .../CURRENT/package/scripts/rest_commands.py| 4 + .../METRON/CURRENT/package/templates/metron.j2 |16 + .../METRON/CURRENT/themes/metron_theme.json | 171 +- .../docker/rpm-docker/SPECS/metron.spec | 5 +- metron-interface/metron-alerts/cypress.json | 7 + .../metron-alerts/cypress/fixtures/config.json |23 + .../cypress/fixtures/pcap.page-01.json | 12383 +++ .../cypress/fixtures/pcap.status-00.json| 7 + .../cypress/fixtures/pcap.status-01.json| 7 + .../cypress/fixtures/pcap.status-02.json| 7 + .../metron-alerts/cypress/fixtures/search.json | 5647 + .../cypress/integration/pcap/pcap.spec.js | 228 + .../metron-alerts/package-lock.json | 1134 + metron-interface/metron-alerts/package.json |11 +- .../alert-details/alert-details.component.ts|10 +- .../alerts/alerts-list/alerts-list.component.ts | 9 +- .../table-view/table-view.component.spec.ts | 4 +- .../table-view/table-view.component.ts |49 +- .../tree-view/tree-view.component.spec.ts | 4 +- .../tree-view/tree-view.component.ts|20 +- .../alerts/meta-alerts/meta-alerts.component.ts | 2 - .../save-search/save-search.component.spec.ts | 4 +- .../alerts/save-search/save-search.component.ts |12 +- .../saved-searches.component.spec.ts| 5 +- .../saved-searches/saved-searches.component.ts |41 +- .../metron-alerts/src/app/app.component.html| 1 + .../metron-alerts/src/app/app.component.spec.ts | 4 + .../metron-alerts/src/app/app.module.ts |12 +- .../src/app/model/confirmation-type.ts
[21/52] [abbrv] metron git commit: METRON-1749 Update Angular to latest release in Management UI (sardell via nickwallen) closes apache/metron#1217
http://git-wip-us.apache.org/repos/asf/metron/blob/0c4c622b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts
--
diff --git
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts
index 647e02f..1ba297c 100644
---
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts
+++
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts
@@ -15,30 +15,36 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import {Component, OnInit, ViewChild} from '@angular/core';
-import {FormGroup, Validators, FormControl} from '@angular/forms';
-import {SensorParserConfig} from '../../model/sensor-parser-config';
-import {SensorParserConfigService} from
'../../service/sensor-parser-config.service';
-import {Router, ActivatedRoute} from '@angular/router';
-import {MetronAlerts} from '../../shared/metron-alerts';
-import {SensorParserContext} from '../../model/sensor-parser-context';
-import {SensorEnrichmentConfigService} from
'../../service/sensor-enrichment-config.service';
-import {SensorEnrichmentConfig} from '../../model/sensor-enrichment-config';
-import {SensorFieldSchemaComponent} from
'../sensor-field-schema/sensor-field-schema.component';
-import {SensorRawJsonComponent} from
'../sensor-raw-json/sensor-raw-json.component';
-import {KafkaService} from '../../service/kafka.service';
-import {SensorIndexingConfigService} from
'../../service/sensor-indexing-config.service';
-import {IndexingConfigurations} from '../../model/sensor-indexing-config';
-import {RestError} from '../../model/rest-error';
-import {HdfsService} from '../../service/hdfs.service';
-import {GrokValidationService} from '../../service/grok-validation.service';
+import { Component, OnInit, ViewChild } from '@angular/core';
+import { FormGroup, Validators, FormControl } from '@angular/forms';
+import { SensorParserConfig } from '../../model/sensor-parser-config';
+import { SensorParserConfigService } from
'../../service/sensor-parser-config.service';
+import { Router, ActivatedRoute } from '@angular/router';
+import { MetronAlerts } from '../../shared/metron-alerts';
+import { SensorParserContext } from '../../model/sensor-parser-context';
+import { SensorEnrichmentConfigService } from
'../../service/sensor-enrichment-config.service';
+import { SensorEnrichmentConfig } from '../../model/sensor-enrichment-config';
+import { SensorFieldSchemaComponent } from
'../sensor-field-schema/sensor-field-schema.component';
+import { SensorRawJsonComponent } from
'../sensor-raw-json/sensor-raw-json.component';
+import { KafkaService } from '../../service/kafka.service';
+import { SensorIndexingConfigService } from
'../../service/sensor-indexing-config.service';
+import { IndexingConfigurations } from '../../model/sensor-indexing-config';
+import { RestError } from '../../model/rest-error';
+import { HdfsService } from '../../service/hdfs.service';
+import { GrokValidationService } from '../../service/grok-validation.service';
export enum Pane {
- GROK, RAWJSON, FIELDSCHEMA, THREATTRIAGE, STORMSETTINGS
+ GROK,
+ RAWJSON,
+ FIELDSCHEMA,
+ THREATTRIAGE,
+ STORMSETTINGS
}
export enum KafkaStatus {
- NO_TOPIC, NOT_EMITTING, EMITTING
+ NO_TOPIC,
+ NOT_EMITTING,
+ EMITTING
}
@Component({
@@ -46,9 +52,7 @@ export enum KafkaStatus {
templateUrl: 'sensor-parser-config.component.html',
styleUrls: ['sensor-parser-config.component.scss']
})
-
export class SensorParserConfigComponent implements OnInit {
-
sensorConfigForm: FormGroup;
transformsValidationForm: FormGroup;
@@ -73,7 +77,7 @@ export class SensorParserConfigComponent implements OnInit {
grokStatementValid = false;
availableParsers = {};
availableParserNames = [];
- grokStatement = '';
+ grokStatement = {};
patternLabel = '';
currentSensors = [];
@@ -81,7 +85,10 @@ export class SensorParserConfigComponent implements OnInit {
topicExists: boolean = false;
- transformsValidationResult: {map: any, keys: string[]} = {map: {}, keys: []};
+ transformsValidationResult: { map: any; keys: string[] } = {
+map: {},
+keys: []
+ };
transformsValidation: SensorParserContext = new SensorParserContext();
pane = Pane;
@@ -90,65 +97,94 @@ export class SensorParserConfigComponent implements OnInit {
kafkaStatus = KafkaStatus;
currentKafkaStatus = null;
- @ViewChild(SensorFieldSchemaComponent) sensorFieldSchema:
SensorFieldSchemaComponent;
- @ViewChild(SensorRawJsonComponent) sensorRawJson: SensorRawJsonComponent;
-
- constructor(private sensorParserConfigService: SensorParserConfigService,
private
[29/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
new file mode 100644
index 000..354322a
--- /dev/null
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
@@ -0,0 +1,388 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.dsl.functions;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.util.EntityUtils;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
+import org.apache.metron.stellar.common.utils.JSONUtils;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import static java.lang.String.format;
+import static org.apache.metron.stellar.dsl.Context.Capabilities.GLOBAL_CONFIG;
+import static
org.apache.metron.stellar.dsl.functions.RestConfig.POOLING_DEFAULT_MAX_PER_RUOTE;
+import static
org.apache.metron.stellar.dsl.functions.RestConfig.POOLING_MAX_TOTAL;
+import static
org.apache.metron.stellar.dsl.functions.RestConfig.STELLAR_REST_SETTINGS;
+
+/**
+ * Defines functions that enable REST requests with proper result and error
handling. Depends on an
+ * Apache HttpComponents client being supplied as a Stellar HTTP_CLIENT
capability. Exposes various Http settings
+ * including authentication, proxy and timeouts through the global config with
the option to override any settings
+ * through a config object supplied in the expression.
+ */
+public class RestFunctions {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * Get an argument from a list of arguments.
+ *
+ * @param index The index within the list of arguments.
+ * @param clazz The type expected.
+ * @param args All of the arguments.
+ * @param The type of the argument expected.
+ */
+ public static T getArg(int index, Class clazz, List args) {
+
+if(index >= args.size()) {
+ throw new IllegalArgumentException(format("Expected at least %d
argument(s), found %d", index+1, args.size()));
+}
+
+return ConversionUtils.convert(args.get(index), clazz);
+ }
+
+ @Stellar(
+ namespace = "REST",
+ name = "GET",
+ description = "Performs a REST GET request and parses the JSON
results into a map.",
+ params = {
+
[41/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/package.json
--
diff --git a/metron-interface/metron-config/package.json
b/metron-interface/metron-config/package.json
index 6e196d5..ffc92ff 100644
--- a/metron-interface/metron-config/package.json
+++ b/metron-interface/metron-config/package.json
@@ -7,10 +7,11 @@
},
"angular-cli": {},
"scripts": {
-"build": "./node_modules/angular-cli/bin/ng build -prod",
+"build": "ng build --prod",
"start": "ng serve",
"lint": "tslint \"src/**/*.ts\"",
-"test": "./node_modules/angular-cli/bin/ng test --watch=false",
+"test": "ng test --browsers=Chrome",
+"testCI": "ng test --watch=false --browsers=ChromeHeadless",
"pree2e": "webdriver-manager update",
"e2e": "./node_modules/.bin/protractor",
"e2e-all": "./node_modules/.bin/protractor --suite=all",
@@ -20,47 +21,54 @@
},
"private": true,
"dependencies": {
+"@angular-devkit/schematics": "^0.7.5",
+"@angular/cli": "^6.2.3",
+"@angular/common": "^6.1.8",
+"@angular/compiler": "^6.1.8",
+"@angular/core": "^6.1.8",
+"@angular/forms": "^6.1.8",
+"@angular/http": "^6.1.8",
+"@angular/platform-browser": "^6.1.8",
+"@angular/platform-browser-dynamic": "^6.1.8",
+"@angular/platform-server": "^6.1.8",
+"@angular/router": "^6.1.8",
"@types/ace": "0.0.32",
-"@types/bootstrap": "^3.3.32",
-"@types/jasmine": "2.2.30",
-"@types/jquery": "^2.0.32",
+"@types/bootstrap": "^4.1.2",
+"@types/jasmine": "~2.8.6",
+"@types/jasminewd2": "~2.0.3",
+"@types/jquery": "^3.3.6",
+"@types/node": "^10.9.4",
"@types/tether": "^1.1.27",
-"@angular/common": "2.0.0",
-"@angular/compiler": "2.0.0",
-"@angular/core": "2.0.0",
-"@angular/forms": "2.0.0",
-"@angular/http": "2.0.0",
-"@angular/platform-browser": "2.0.0",
-"@angular/platform-browser-dynamic": "2.0.0",
-"@angular/router": "3.0.0",
"ace-builds": "^1.2.5",
"bootstrap": "4.0.0-alpha.5",
-"core-js": "^2.4.1",
+"core-js": "^2.5.7",
"font-awesome": "^4.6.3",
"jquery": "^3.3.1",
-"rxjs": "5.0.0-beta.12",
+"karma-phantomjs-launcher": "^1.0.4",
+"puppeteer": "^1.8.0",
+"rxjs": "6.2.2",
"tether": "^1.3.4",
"ts-helpers": "^1.1.1",
-"zone.js": "^0.6.23"
+"zone.js": "^0.8.26"
},
"devDependencies": {
-"angular-cli": "1.0.0-beta.15",
+"@angular-devkit/build-angular": "^0.8.3",
+"@angular/compiler-cli": "^6.1.8",
+"@types/request": "2.0.3",
"buffer-shims": "^1.0.0",
-"codelyzer": "~0.0.26",
+"codelyzer": "~4.2.1",
"copy": "^0.3.0",
-"jasmine-core": "2.4.1",
-"jasmine-spec-reporter": "2.5.0",
-"karma": "1.2.0",
-"karma-chrome-launcher": "^2.0.0",
-"karma-cli": "^1.0.1",
-"karma-jasmine": "^1.0.2",
-"karma-phantomjs-launcher": "^1.0.4",
-"karma-remap-istanbul": "0.6.0",
+"jasmine-core": "~2.99.1",
+"jasmine-spec-reporter": "~4.2.1",
+"karma": "^3.0.0",
+"karma-chrome-launcher": "~2.2.0",
+"karma-coverage-istanbul-reporter": "~2.0.0",
+"karma-jasmine": "^1.1.2",
+"karma-jasmine-html-reporter": "^0.2.2",
"phantomjs-prebuilt": "^2.1.14",
-"protractor": "4.0.5",
-"ts-node": "1.2.1",
-"tslint": "3.13.0",
-"typescript": "~2.0.3",
-"@types/request": "2.0.3"
+"protractor": "^5.4.1",
+"ts-node": "~5.0.1",
+"tslint": "^5.11.0",
+"typescript": "~2.7.2"
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/pom.xml
--
diff --git a/metron-interface/metron-config/pom.xml
b/metron-interface/metron-config/pom.xml
index fd76447..c6ef5b5 100644
--- a/metron-interface/metron-config/pom.xml
+++ b/metron-interface/metron-config/pom.xml
@@ -63,7 +63,7 @@
generate-resources
-ng build
+npm run build
npm
@@ -72,13 +72,13 @@
-npm test
+test
+npm testCI
- npm
+npm
-test
- test
+run testCI
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/scripts/prepend_license_header.sh
--
diff --git a/metron-interface/metron-config/scripts/prepend_license_header.sh
b/metron-interface/metron-config/scripts/prepend_license_header.sh
index 1957cd6..6bf004f 100755
--- a/metron-interface/metron-config/scripts/prepend_license_header.sh
[10/52] [abbrv] metron git commit: METRON-1861 METRON-1861: REST fails to start when LDAP enabled and 'Active Spring profiles' config is empty (anandsubbu via justinleet) closes apache/metron#1256
METRON-1861 METRON-1861: REST fails to start when LDAP enabled and 'Active Spring profiles' config is empty (anandsubbu via justinleet) closes apache/metron#1256 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7bba8be0 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7bba8be0 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7bba8be0 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 7bba8be0cb77136459c25af697163d76fcf45100 Parents: 85cd21a Author: anandsubbu Authored: Thu Nov 8 09:08:39 2018 -0500 Committer: leet Committed: Thu Nov 8 09:08:39 2018 -0500 -- .../METRON/CURRENT/package/scripts/params/params_linux.py | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/7bba8be0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 75f68fc..060dfe4 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -62,7 +62,10 @@ metron_jvm_flags = config['configurations']['metron-rest-env']['metron_jvm_flags # Construct the profiles as a temp variable first. Only the first time it's set will carry through metron_spring_profiles_temp = config['configurations']['metron-rest-env']['metron_spring_profiles_active'] if config['configurations']['metron-security-env']['metron.ldap.enabled']: -metron_spring_profiles_active = metron_spring_profiles_temp + ',ldap' +if metron_spring_profiles_temp: +metron_spring_profiles_active = metron_spring_profiles_temp + ',ldap' +else: +metron_spring_profiles_active = 'ldap' else: metron_spring_profiles_active = metron_spring_profiles_temp
[15/52] [abbrv] metron git commit: METRON-1740 Improve Palo Alto parser to handle CONFIG and SYSTEM syslog messages (liuy-tnz via nickwallen) closes apache/metron#1171
METRON-1740 Improve Palo Alto parser to handle CONFIG and SYSTEM syslog
messages (liuy-tnz via nickwallen) closes apache/metron#1171
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/61b1ede0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/61b1ede0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/61b1ede0
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 61b1ede0bdfd28b04a3783371deb935d293df998
Parents: 42068d9
Author: liuy-tnz
Authored: Tue Nov 13 07:00:51 2018 -0500
Committer: nickallen
Committed: Tue Nov 13 07:00:51 2018 -0500
--
.../paloalto/BasicPaloAltoFirewallParser.java | 413 ---
.../apache/metron/parsers/paloalto/README.md| 32 ++
.../BasicPaloAltoFirewallParserTest.java| 244 +++
3 files changed, 541 insertions(+), 148 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/61b1ede0/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
--
diff --git
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
index 9051f09..c8e8b62 100644
---
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
+++
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
@@ -34,21 +34,26 @@ import java.util.regex.Pattern;
public class BasicPaloAltoFirewallParser extends BasicParser {
- private static boolean empty_attribute( final String s ) {
+ private static boolean empty_attribute(final String s) {
return s == null || s.trim().isEmpty() || s.equals("\"\"");
}
- private static String unquoted_attribute( String s ) {
+ private static String unquoted_attribute(String s) {
s = s.trim();
-if ( s.startsWith( "\"" ) && s.endsWith( "\"" ) )
- return s.substring( 1, s.length( ) - 1 );
+if (s.startsWith("\"") && s.endsWith("\""))
+ return s.substring(1, s.length() - 1);
return s;
}
- private static final Logger _LOG = LoggerFactory.getLogger
- (BasicPaloAltoFirewallParser.class);
+ private static final Logger _LOG =
LoggerFactory.getLogger(BasicPaloAltoFirewallParser.class);
private static final long serialVersionUID = 3147090149725343999L;
+
+ private static final String LogTypeConfig = "CONFIG";
+ private static final String LogTypeSystem = "SYSTEM";
+ private static final String LogTypeThreat = "THREAT";
+ private static final String LogTypeTraffic = "TRAFFIC";
+
public static final String PaloAltoDomain = "palo_alto_domain";
public static final String ReceiveTime = "receive_time";
public static final String SerialNum = "serial";
@@ -101,6 +106,21 @@ public class BasicPaloAltoFirewallParser extends
BasicParser {
public static final String ParentSessionStartTime =
"parent_session_start_time";
public static final String TunnelType = "tunnel_type";
+ //System
+ public static final String EventId = "event_id";
+ public static final String Object = "object";
+ public static final String Module = "module";
+ public static final String Description = "description";
+
+ //Config
+ public static final String Command = "command";
+ public static final String Admin = "admin";
+ public static final String Client = "client";
+ public static final String Result = "result";
+ public static final String ConfigurationPath = "configuration_path";
+ public static final String BeforeChangeDetail = "before_change_detail";
+ public static final String AfterChangeDetail = "after_change_detail";
+
//Threat
public static final String URL = "url";
public static final String HOST = "host";
@@ -113,7 +133,7 @@ public class BasicPaloAltoFirewallParser extends
BasicParser {
public static final String PCAPID = "pcap_id";
public static final String WFFileDigest = "filedigest";
public static final String WFCloud = "cloud";
- public static final String UserAgent= "user_agent";
+ public static final String UserAgent = "user_agent";
public static final String WFFileType = "filetype";
public static final String XForwardedFor = "xff";
public static final String Referer = "referer";
@@ -159,8 +179,6 @@ public class BasicPaloAltoFirewallParser extends
BasicParser {
toParse = new String(msg, "UTF-8");
_LOG.debug("Received message: {}", toParse);
-
-
parseMessage(toParse, outputMessage);
long timestamp = System.currentTimeMillis();
outputMessage.put("timestamp",
[38/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts
--
diff --git
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts
index 647e02f..1ba297c 100644
---
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts
+++
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts
@@ -15,30 +15,36 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import {Component, OnInit, ViewChild} from '@angular/core';
-import {FormGroup, Validators, FormControl} from '@angular/forms';
-import {SensorParserConfig} from '../../model/sensor-parser-config';
-import {SensorParserConfigService} from
'../../service/sensor-parser-config.service';
-import {Router, ActivatedRoute} from '@angular/router';
-import {MetronAlerts} from '../../shared/metron-alerts';
-import {SensorParserContext} from '../../model/sensor-parser-context';
-import {SensorEnrichmentConfigService} from
'../../service/sensor-enrichment-config.service';
-import {SensorEnrichmentConfig} from '../../model/sensor-enrichment-config';
-import {SensorFieldSchemaComponent} from
'../sensor-field-schema/sensor-field-schema.component';
-import {SensorRawJsonComponent} from
'../sensor-raw-json/sensor-raw-json.component';
-import {KafkaService} from '../../service/kafka.service';
-import {SensorIndexingConfigService} from
'../../service/sensor-indexing-config.service';
-import {IndexingConfigurations} from '../../model/sensor-indexing-config';
-import {RestError} from '../../model/rest-error';
-import {HdfsService} from '../../service/hdfs.service';
-import {GrokValidationService} from '../../service/grok-validation.service';
+import { Component, OnInit, ViewChild } from '@angular/core';
+import { FormGroup, Validators, FormControl } from '@angular/forms';
+import { SensorParserConfig } from '../../model/sensor-parser-config';
+import { SensorParserConfigService } from
'../../service/sensor-parser-config.service';
+import { Router, ActivatedRoute } from '@angular/router';
+import { MetronAlerts } from '../../shared/metron-alerts';
+import { SensorParserContext } from '../../model/sensor-parser-context';
+import { SensorEnrichmentConfigService } from
'../../service/sensor-enrichment-config.service';
+import { SensorEnrichmentConfig } from '../../model/sensor-enrichment-config';
+import { SensorFieldSchemaComponent } from
'../sensor-field-schema/sensor-field-schema.component';
+import { SensorRawJsonComponent } from
'../sensor-raw-json/sensor-raw-json.component';
+import { KafkaService } from '../../service/kafka.service';
+import { SensorIndexingConfigService } from
'../../service/sensor-indexing-config.service';
+import { IndexingConfigurations } from '../../model/sensor-indexing-config';
+import { RestError } from '../../model/rest-error';
+import { HdfsService } from '../../service/hdfs.service';
+import { GrokValidationService } from '../../service/grok-validation.service';
export enum Pane {
- GROK, RAWJSON, FIELDSCHEMA, THREATTRIAGE, STORMSETTINGS
+ GROK,
+ RAWJSON,
+ FIELDSCHEMA,
+ THREATTRIAGE,
+ STORMSETTINGS
}
export enum KafkaStatus {
- NO_TOPIC, NOT_EMITTING, EMITTING
+ NO_TOPIC,
+ NOT_EMITTING,
+ EMITTING
}
@Component({
@@ -46,9 +52,7 @@ export enum KafkaStatus {
templateUrl: 'sensor-parser-config.component.html',
styleUrls: ['sensor-parser-config.component.scss']
})
-
export class SensorParserConfigComponent implements OnInit {
-
sensorConfigForm: FormGroup;
transformsValidationForm: FormGroup;
@@ -73,7 +77,7 @@ export class SensorParserConfigComponent implements OnInit {
grokStatementValid = false;
availableParsers = {};
availableParserNames = [];
- grokStatement = '';
+ grokStatement = {};
patternLabel = '';
currentSensors = [];
@@ -81,7 +85,10 @@ export class SensorParserConfigComponent implements OnInit {
topicExists: boolean = false;
- transformsValidationResult: {map: any, keys: string[]} = {map: {}, keys: []};
+ transformsValidationResult: { map: any; keys: string[] } = {
+map: {},
+keys: []
+ };
transformsValidation: SensorParserContext = new SensorParserContext();
pane = Pane;
@@ -90,65 +97,94 @@ export class SensorParserConfigComponent implements OnInit {
kafkaStatus = KafkaStatus;
currentKafkaStatus = null;
- @ViewChild(SensorFieldSchemaComponent) sensorFieldSchema:
SensorFieldSchemaComponent;
- @ViewChild(SensorRawJsonComponent) sensorRawJson: SensorRawJsonComponent;
-
- constructor(private sensorParserConfigService: SensorParserConfigService,
private
[16/52] [abbrv] metron git commit: METRON-1868 metron-committer-common incorrectly checking REPO_NAME (JonZeolla via jonzeolla) closes apache/metron#1260
METRON-1868 metron-committer-common incorrectly checking REPO_NAME (JonZeolla
via jonzeolla) closes apache/metron#1260
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0303d240
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0303d240
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0303d240
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 0303d240573a96d0bc007f3c753bc8f8fca5b308
Parents: 61b1ede
Author: JonZeolla
Authored: Wed Nov 14 10:25:19 2018 -0500
Committer: jonzeolla
Committed: Wed Nov 14 10:25:19 2018 -0500
--
dev-utilities/committer-utils/metron-committer-common | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/0303d240/dev-utilities/committer-utils/metron-committer-common
--
diff --git a/dev-utilities/committer-utils/metron-committer-common
b/dev-utilities/committer-utils/metron-committer-common
index 22e19a2..0f6c06a 100644
--- a/dev-utilities/committer-utils/metron-committer-common
+++ b/dev-utilities/committer-utils/metron-committer-common
@@ -43,6 +43,7 @@ DESC=
APACHE_NAME=
APACHE_EMAIL=
GITHUB_NAME=
+CHOSEN_REPO=
#
# Initialize the variables from the default configuration file, if it exists
@@ -174,7 +175,7 @@ function setup_code {
if [ ! -d "$WORK" ]; then
REPO_NAME="metron"
-if [ -z $1 ]; then
+if [ -n $1 ]; then
REPO_NAME=$1
fi
@@ -354,4 +355,4 @@ function please_review_then {
echo "cd $WORK"
echo "git push upstream $BASE_BRANCH"
echo ""
-}
\ No newline at end of file
+}
[12/52] [abbrv] metron git commit: METRON-1858 BasicFireEyeParser check style cleanup and optimization (ottobackwards) closes apache/metron#1255
METRON-1858 BasicFireEyeParser check style cleanup and optimization
(ottobackwards) closes apache/metron#1255
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b5712af5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b5712af5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b5712af5
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: b5712af5b78ff8c6e5e3d3f3c68552204fa7d877
Parents: c6b1b3c
Author: ottobackwards
Authored: Thu Nov 8 14:14:03 2018 -0500
Committer: otto
Committed: Thu Nov 8 14:14:03 2018 -0500
--
.../parsers/fireeye/BasicFireEyeParser.java | 347 +--
1 file changed, 158 insertions(+), 189 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/b5712af5/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
--
diff --git
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
index 489eb00..1cf9fb7 100644
---
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
+++
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,18 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.metron.parsers.fireeye;
import com.google.common.base.Joiner;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.metron.parsers.utils.ParserUtils;
-import org.apache.metron.parsers.BasicParser;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -34,189 +30,162 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.parsers.BasicParser;
+import org.apache.metron.parsers.utils.ParserUtils;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public class BasicFireEyeParser extends BasicParser {
-
- private static final long serialVersionUID = 6328907550159134550L;
- protected static final Logger LOG = LoggerFactory
- .getLogger(BasicFireEyeParser.class);
-
-
- String tsRegex
="([a-zA-Z]{3})\\s+(\\d+)\\s+(\\d+\\:\\d+\\:\\d+)\\s+(\\d+\\.\\d+\\.\\d+\\.\\d+)";
-
-
- Pattern tsPattern = Pattern.compile(tsRegex);
- // private transient static MetronGrok grok;
- // private transient static InputStream pattern_url;
-
- public BasicFireEyeParser() throws Exception {
- // pattern_url =
getClass().getClassLoader().getResourceAsStream(
- // "patterns/fireeye");
- //
- // File file = ParserUtils.stream2file(pattern_url);
- // grok = MetronGrok.create(file.getPath());
- //
- // grok.compile("%{FIREEYE_BASE}");
- }
-
- @Override
- public void configure(Map parserConfig) {
-
- }
-
- @Override
- public void init() {
-
- }
-
- @Override
- public List parse(byte[] raw_message) {
- String toParse = "";
- List messages = new ArrayList<>();
- try {
-
- toParse = new String(raw_message, "UTF-8");
-
- // String[] mTokens = toParse.split(" ");
-
- String positveIntPattern = "<[1-9][0-9]*>";
- Pattern p = Pattern.compile(positveIntPattern);
- Matcher m = p.matcher(toParse);
-
- String delimiter = "";
-
- while (m.find()) {
- delimiter = m.group();
-
- }
-
- if (!StringUtils.isBlank(delimiter)) {
- String[] tokens = toParse.split(delimiter);
-
- i
[27/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (cstella via mmiklavc)
http://git-wip-us.apache.org/repos/asf/metron/blob/e7e19fbb/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 8071e68..61dd0f6 100644
---
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -39,6 +39,7 @@ import org.apache.metron.indexing.dao.search.SearchRequest;
import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.integration.InMemoryComponent;
+import org.apache.metron.integration.utils.TestUtils;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -200,7 +201,7 @@ public class ElasticsearchSearchIntegrationTest extends
SearchIntegrationTest {
config.setGlobalConfigSupplier( () ->
new HashMap() {{
put("es.clustername", "metron");
- put("es.port", "9300");
+ put("es.port", "9200");
put("es.ip", "localhost");
put("es.date.format", dateFormat);
}}
@@ -272,8 +273,10 @@ public class ElasticsearchSearchIntegrationTest extends
SearchIntegrationTest {
public void returns_column_metadata_for_specified_indices() throws Exception
{
// getColumnMetadata with only bro
{
+ //TODO: It shouldn't require an assertEventually() here as it should be
synchronous.
+ // Before merging, please figure out why.
+ TestUtils.assertEventually(() -> Assert.assertEquals(13,
dao.getColumnMetadata(Collections.singletonList("bro")).size()));
Map fieldTypes =
dao.getColumnMetadata(Collections.singletonList("bro"));
- Assert.assertEquals(13, fieldTypes.size());
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("ttl"));
Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
@@ -292,8 +295,10 @@ public class ElasticsearchSearchIntegrationTest extends
SearchIntegrationTest {
}
// getColumnMetadata with only snort
{
+ //TODO: It shouldn't require an assertEventually() here as it should be
synchronous.
+ // Before merging, please figure out why.
+ TestUtils.assertEventually(() -> Assert.assertEquals(14,
dao.getColumnMetadata(Collections.singletonList("snort")).size()));
Map fieldTypes =
dao.getColumnMetadata(Collections.singletonList("snort"));
- Assert.assertEquals(14, fieldTypes.size());
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ttl"));
Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
@@ -313,8 +318,10 @@ public class ElasticsearchSearchIntegrationTest extends
SearchIntegrationTest {
@Override
public void returns_column_data_for_multiple_indices() throws Exception {
+//TODO: It shouldn't require an assertEventually() here as it should be
synchronous.
+// Before merging, please figure out why.
+TestUtils.assertEventually(() -> Assert.assertEquals(15,
dao.getColumnMetadata(Arrays.asList("bro", "snort")).size()));
Map fieldTypes =
dao.getColumnMetadata(Arrays.asList("bro", "snort"));
-Assert.assertEquals(15, fieldTypes.size());
Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
http://git-wip-us.apache.org/repos/asf/metron/blob/e7e19fbb/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
index c5c0bc1..6f36790 100644
---
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
+++
[43/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-alerts/src/app/alerts/saved-searches/saved-searches.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/saved-searches/saved-searches.component.spec.ts
b/metron-interface/metron-alerts/src/app/alerts/saved-searches/saved-searches.component.spec.ts
index 899a2b1..a953b32 100644
---
a/metron-interface/metron-alerts/src/app/alerts/saved-searches/saved-searches.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/saved-searches/saved-searches.component.spec.ts
@@ -16,7 +16,6 @@
* limitations under the License.
*/
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
-import { Component, Input } from '@angular/core';
import { SavedSearchesComponent } from './saved-searches.component';
import { CollapseComponent } from '../../shared/collapse/collapse.component';
@@ -24,8 +23,8 @@ import { CenterEllipsesPipe } from
'../../shared/pipes/center-ellipses.pipe';
import { ColumnNameTranslatePipe } from
'../../shared/pipes/column-name-translate.pipe';
import { Router } from '@angular/router';
import { SaveSearchService } from '../../service/save-search.service';
-import { MetronDialogBox } from '../../shared/metron-dialog-box';
import { of } from 'rxjs';
+import { DialogService } from 'app/service/dialog.service';
describe('SavedSearchesComponent', () => {
@@ -40,7 +39,7 @@ describe('SavedSearchesComponent', () => {
listSavedSearches:
jasmine.createSpy('listSavedSearches').and.returnValue(of([])),
listRecentSearches:
jasmine.createSpy('listRecentSearches').and.returnValue(of([])),
} },
-MetronDialogBox
+DialogService
],
declarations: [
SavedSearchesComponent,
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-alerts/src/app/alerts/saved-searches/saved-searches.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/saved-searches/saved-searches.component.ts
b/metron-interface/metron-alerts/src/app/alerts/saved-searches/saved-searches.component.ts
index 2204179..ab182c0 100644
---
a/metron-interface/metron-alerts/src/app/alerts/saved-searches/saved-searches.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/saved-searches/saved-searches.component.ts
@@ -22,9 +22,10 @@ import {forkJoin as observableForkJoin} from 'rxjs';
import {SaveSearchService} from '../../service/save-search.service';
import {SaveSearch} from '../../model/save-search';
-import {MetronDialogBox} from '../../shared/metron-dialog-box';
import {NUM_SAVED_SEARCH} from '../../utils/constants';
import {CollapseComponentData, CollapseComponentDataItems} from
'../../shared/collapse/collapse-component-data';
+import { DialogService } from 'app/service/dialog.service';
+import { ConfirmationType } from 'app/model/confirmation-type';
@Component({
selector: 'app-saved-searches',
@@ -39,7 +40,7 @@ export class SavedSearchesComponent implements OnInit {
recentSearches: CollapseComponentData = new CollapseComponentData();
constructor(private router: Router,
private saveSearchService: SaveSearchService,
- private metronDialog: MetronDialogBox) {
+ private dialogService: DialogService) {
}
doDeleteRecentSearch(selectedSearch: SaveSearch) {
@@ -61,21 +62,33 @@ export class SavedSearchesComponent implements OnInit {
}
deleteRecentSearch($event) {
-let selectedSearch = this.recentSearcheObj.find(savedSearch =>
savedSearch.name === $event.key);
-this.metronDialog.showConfirmationMessage('Do you wish to delete recent
search ' + selectedSearch.name).subscribe((result: boolean) => {
- if (result) {
-this.doDeleteRecentSearch(selectedSearch);
- }
-});
+let selectedSearch = this.recentSearcheObj.find(
+ savedSearch => savedSearch.name === $event.key
+);
+const confirmedSubscription = this.dialogService
+ .launchDialog(
+'Do you wish to delete recent search ' + selectedSearch.name
+ )
+ .subscribe(action => {
+if (action === ConfirmationType.Confirmed) {
+ this.doDeleteRecentSearch(selectedSearch);
+}
+confirmedSubscription.unsubscribe();
+ });
}
deleteSearch($event) {
-let selectedSearch = this.searches.find(savedSearch => savedSearch.name
=== $event.key);
-this.metronDialog.showConfirmationMessage('Do you wish to delete saved
search ' + selectedSearch.name).subscribe((result: boolean) => {
- if (result) {
-this.doDeleteSearch(selectedSearch);
- }
-});
+let selectedSearch = this.searches.find(
+ savedSearch => savedSearch.name === $event.key
+);
+const confirmedSubscription =
[26/52] [abbrv] metron git commit: METRON-1749 Update Angular to latest release in Management UI (sardell via nickwallen) closes apache/metron#1217
METRON-1749 Update Angular to latest release in Management UI (sardell via nickwallen) closes apache/metron#1217 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0c4c622b Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0c4c622b Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0c4c622b Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 0c4c622b927d1ba3161779dc2e745d5b74ae06b6 Parents: 0e2a434 Author: sardell Authored: Wed Nov 14 15:02:53 2018 -0500 Committer: nickallen Committed: Wed Nov 14 15:02:53 2018 -0500 -- .../docker/rpm-docker/SPECS/metron.spec | 5 +- metron-interface/metron-config/angular-cli.json |51 - metron-interface/metron-config/angular.json | 155 + metron-interface/metron-config/karma.conf.js|60 +- .../metron-config/package-lock.json | 18987 ++--- metron-interface/metron-config/package.json |70 +- metron-interface/metron-config/pom.xml |10 +- .../scripts/prepend_license_header.sh | 2 +- .../metron-config/scripts/start_dev.sh | 2 +- .../metron-config/src/app/_fonts.scss | 101 + .../metron-config/src/app/_main.scss| 112 - .../metron-config/src/app/app.component.spec.ts |64 +- .../metron-config/src/app/app.component.ts | 3 +- .../metron-config/src/app/app.config.ts | 4 +- .../metron-config/src/app/app.module.ts |17 +- .../metron-config/src/app/app.routes.ts |10 +- .../general-settings.component.spec.ts |85 +- .../general-settings.component.ts |48 +- .../metron-config/src/app/global.scss | 2 +- .../default-headers.interceptor.ts |32 + .../src/app/login/login.component.spec.ts |42 +- .../metron-config/src/app/model/rest-error.ts | 4 +- .../src/app/navbar/navbar.component.spec.ts |19 +- .../src/app/navbar/navbar.component.ts |31 +- .../metron-config/src/app/navbar/navbar.html| 2 +- .../metron-config/src/app/rxjs-operators.ts |32 - .../sensor-field-schema.component.html | 2 +- .../sensor-field-schema.component.spec.ts | 1088 +- .../sensor-grok/sensor-grok.component.spec.ts | 137 +- .../sensor-grok/sensor-grok.component.ts| 2 +- ...sor-parser-config-readonly.component.spec.ts | 430 +- .../sensor-parser-config-readonly.component.ts | 389 +- .../sensor-parser-config.component.spec.ts | 615 +- .../sensor-parser-config.component.ts | 442 +- .../sensor-parser-list.component.spec.ts| 512 +- .../sensor-raw-json.component.spec.ts | 345 +- .../sensor-storm-settings.component.spec.ts | 294 +- .../sensor-threat-triage.component.spec.ts | 129 +- .../app/service/authentication.service.spec.ts | 276 +- .../src/app/service/authentication.service.ts |73 +- .../app/service/global-config.service.spec.ts | 101 +- .../src/app/service/global-config.service.ts|25 +- .../app/service/grok-validation.service.spec.ts | 122 +- .../src/app/service/grok-validation.service.ts |53 +- .../src/app/service/hdfs.service.spec.ts| 129 +- .../src/app/service/hdfs.service.ts |63 +- .../src/app/service/kafka.service.spec.ts | 123 +- .../src/app/service/kafka.service.ts|57 +- .../sensor-enrichment-config.service.spec.ts| 240 +- .../service/sensor-enrichment-config.service.ts |76 +- .../sensor-indexing-config.service.spec.ts | 150 +- .../service/sensor-indexing-config.service.ts |62 +- ...sensor-parser-config-history.service.spec.ts | 107 +- .../sensor-parser-config-history.service.ts |55 +- .../sensor-parser-config.service.spec.ts| 206 +- .../app/service/sensor-parser-config.service.ts | 130 +- .../src/app/service/stellar.service.spec.ts | 187 +- .../src/app/service/stellar.service.ts |72 +- .../src/app/service/storm.service.spec.ts | 402 +- .../src/app/service/storm.service.ts| 156 +- .../shared/ace-editor/ace-editor.component.ts | 9 +- .../metron-config/src/app/shared/auth-guard.ts | 2 +- .../src/app/shared/metron-dialog-box.ts | 6 +- .../sample-data/sample-data.component.html | 2 +- .../sample-data/sample-data.component.spec.ts |41 +- .../metron-config/src/app/util/httpUtil.ts |24 +- .../src/app/util/httpUtils.spec.ts |57 +- .../verticalnavbar.component.spec.ts|30 +- .../verticalnavbar/verticalnavbar.component.ts | 3 - .../src/app/verticalnavbar/verticalnavbar.html | 4 +- metron-interface/metron-config/src/styles.scss | 2 +-
[11/52] [abbrv] metron git commit: METRON-1864 Stellar date format test fails after daylight saving (ottobackwards) closes apache/metron#1258
METRON-1864 Stellar date format test fails after daylight saving
(ottobackwards) closes apache/metron#1258
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c6b1b3cc
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c6b1b3cc
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c6b1b3cc
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: c6b1b3ccd7f937342c5723ca0a952a4900efc5cf
Parents: 7bba8be
Author: ottobackwards
Authored: Thu Nov 8 13:51:38 2018 -0500
Committer: otto
Committed: Thu Nov 8 13:51:38 2018 -0500
--
.../metron/stellar/dsl/functions/DateFunctionsTest.java | 12 ++--
1 file changed, 10 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/c6b1b3cc/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
--
diff --git
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
index 959e7ea..48b2995 100644
---
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
+++
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
@@ -28,6 +28,8 @@ import org.apache.metron.stellar.dsl.StellarFunctions;
import org.junit.Before;
import org.junit.Test;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Map;
@@ -233,10 +235,16 @@ public class DateFunctionsTest {
assertEquals("Thu Aug 25 2016 08:27:10 EST", result);
}
+ /**
+ * Test that the String returned is formatted as specified.
+ * LocalDate.parse will throw if it is not.
+ * @throws Exception
+ */
@Test
- public void testDateFormatDefault() {
+ public void testDateFormatDefault() throws Exception {
Object result = run("DATE_FORMAT('EEE MMM dd hh:mm:ss ')");
-
assertTrue(result.toString().endsWith(TimeZone.getDefault().getDisplayName(true,
1)));
+DateTimeFormatter formatter = DateTimeFormatter.ofPattern("EEE MMM dd
hh:mm:ss ");
+LocalDate.parse(result.toString(), formatter);
}
@Test
[19/52] [abbrv] metron git commit: METRON-1749 Update Angular to latest release in Management UI (sardell via nickwallen) closes apache/metron#1217
http://git-wip-us.apache.org/repos/asf/metron/blob/0c4c622b/metron-interface/metron-config/src/app/service/sensor-enrichment-config.service.ts
--
diff --git
a/metron-interface/metron-config/src/app/service/sensor-enrichment-config.service.ts
b/metron-interface/metron-config/src/app/service/sensor-enrichment-config.service.ts
index 90c314b..bc26581 100644
---
a/metron-interface/metron-config/src/app/service/sensor-enrichment-config.service.ts
+++
b/metron-interface/metron-config/src/app/service/sensor-enrichment-config.service.ts
@@ -15,57 +15,69 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import {Injectable, Inject} from '@angular/core';
-import {Http, Headers, RequestOptions, Response} from '@angular/http';
-import {Observable} from 'rxjs/Observable';
-import {SensorEnrichmentConfig} from '../model/sensor-enrichment-config';
-import {HttpUtil} from '../util/httpUtil';
-import {IAppConfig} from '../app.config.interface';
-import {APP_CONFIG} from '../app.config';
+import { Injectable, Inject } from '@angular/core';
+import { HttpClient, HttpResponse } from '@angular/common/http';
+import { Observable } from 'rxjs';
+import { map, catchError } from 'rxjs/operators';
+import { SensorEnrichmentConfig } from '../model/sensor-enrichment-config';
+import { HttpUtil } from '../util/httpUtil';
+import { IAppConfig } from '../app.config.interface';
+import { APP_CONFIG } from '../app.config';
@Injectable()
export class SensorEnrichmentConfigService {
url = this.config.apiEndpoint + '/sensor/enrichment/config';
- defaultHeaders = {'Content-Type': 'application/json', 'X-Requested-With':
'XMLHttpRequest'};
- constructor(private http: Http, @Inject(APP_CONFIG) private config:
IAppConfig) {
- }
+ constructor(
+private http: HttpClient,
+@Inject(APP_CONFIG) private config: IAppConfig
+ ) {}
- public post(name: string, sensorEnrichmentConfig: SensorEnrichmentConfig):
Observable {
-return this.http.post(this.url + '/' + name,
JSON.stringify(sensorEnrichmentConfig),
- new RequestOptions({headers: new
Headers(this.defaultHeaders)}))
- .map(HttpUtil.extractData)
- .catch(HttpUtil.handleError);
+ public post(
+name: string,
+sensorEnrichmentConfig: SensorEnrichmentConfig
+ ): Observable {
+return this.http
+ .post(this.url + '/' + name, JSON.stringify(sensorEnrichmentConfig))
+ .pipe(
+map(HttpUtil.extractData),
+catchError(HttpUtil.handleError)
+ );
}
public get(name: string): Observable {
-return this.http.get(this.url + '/' + name, new RequestOptions({headers:
new Headers(this.defaultHeaders)}))
- .map(HttpUtil.extractData)
- .catch(HttpUtil.handleError);
+return this.http.get(this.url + '/' + name).pipe(
+ map(HttpUtil.extractData),
+ catchError(HttpUtil.handleError)
+);
}
public getAll(): Observable {
-return this.http.get(this.url, new RequestOptions({headers: new
Headers(this.defaultHeaders)}))
- .map(HttpUtil.extractData)
- .catch(HttpUtil.handleError);
+return this.http.get(this.url).pipe(
+ map(HttpUtil.extractData),
+ catchError(HttpUtil.handleError)
+);
}
- public deleteSensorEnrichments(name: string): Observable {
-return this.http.delete(this.url + '/' + name, new
RequestOptions({headers: new Headers(this.defaultHeaders)}))
- .catch(HttpUtil.handleError);
+ public deleteSensorEnrichments(name: string) {
+return this.http
+ .delete>(this.url + '/' + name)
+ .pipe>(catchError(HttpUtil.handleError));
}
public getAvailableEnrichments(): Observable {
-return this.http.get(this.url + '/list/available/enrichments', new
RequestOptions({headers: new Headers(this.defaultHeaders)}))
-.map(HttpUtil.extractData)
-.catch(HttpUtil.handleError);
+return this.http.get(this.url + '/list/available/enrichments').pipe(
+ map(HttpUtil.extractData),
+ catchError(HttpUtil.handleError)
+);
}
public getAvailableThreatTriageAggregators(): Observable {
-return this.http.get(this.url +
'/list/available/threat/triage/aggregators',
-new RequestOptions({headers: new Headers(this.defaultHeaders)}))
-.map(HttpUtil.extractData)
-.catch(HttpUtil.handleError);
+return this.http
+ .get(this.url + '/list/available/threat/triage/aggregators')
+ .pipe(
+map(HttpUtil.extractData),
+catchError(HttpUtil.handleError)
+ );
}
-
}
http://git-wip-us.apache.org/repos/asf/metron/blob/0c4c622b/metron-interface/metron-config/src/app/service/sensor-indexing-config.service.spec.ts
--
diff --git
a/metron-interface/metron-config/src/app/service/sensor-indexing-config.service.spec.ts
[39/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts -- diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts b/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts index 5db6d45..2904ef5 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts @@ -15,23 +15,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import {Component, OnInit} from '@angular/core'; -import {KafkaService} from '../../service/kafka.service'; -import {Router, ActivatedRoute} from '@angular/router'; -import {KafkaTopic} from '../../model/kafka-topic'; -import {MetronAlerts} from '../../shared/metron-alerts'; -import {SensorParserConfigService} from '../../service/sensor-parser-config.service'; -import {StormService} from '../../service/storm.service'; -import {TopologyStatus} from '../../model/topology-status'; -import {SensorParserConfigHistoryService} from '../../service/sensor-parser-config-history.service'; -import {SensorParserConfigHistory} from '../../model/sensor-parser-config-history'; -import {SensorEnrichmentConfigService} from '../../service/sensor-enrichment-config.service'; -import {SensorEnrichmentConfig} from '../../model/sensor-enrichment-config'; -import {RiskLevelRule} from '../../model/risk-level-rule'; -import {HdfsService} from '../../service/hdfs.service'; -import {RestError} from '../../model/rest-error'; -import {GrokValidationService} from '../../service/grok-validation.service'; -import {SensorParserConfig} from '../../model/sensor-parser-config'; +import { Component, OnInit } from '@angular/core'; +import { KafkaService } from '../../service/kafka.service'; +import { Router, ActivatedRoute } from '@angular/router'; +import { KafkaTopic } from '../../model/kafka-topic'; +import { MetronAlerts } from '../../shared/metron-alerts'; +import { SensorParserConfigService } from '../../service/sensor-parser-config.service'; +import { StormService } from '../../service/storm.service'; +import { TopologyStatus } from '../../model/topology-status'; +import { SensorParserConfigHistoryService } from '../../service/sensor-parser-config-history.service'; +import { SensorParserConfigHistory } from '../../model/sensor-parser-config-history'; +import { SensorEnrichmentConfigService } from '../../service/sensor-enrichment-config.service'; +import { SensorEnrichmentConfig } from '../../model/sensor-enrichment-config'; +import { RiskLevelRule } from '../../model/risk-level-rule'; +import { HdfsService } from '../../service/hdfs.service'; +import { RestError } from '../../model/rest-error'; +import { GrokValidationService } from '../../service/grok-validation.service'; +import { SensorParserConfig } from '../../model/sensor-parser-config'; @Component({ selector: 'metron-config-sensor-parser-readonly', @@ -39,7 +39,6 @@ import {SensorParserConfig} from '../../model/sensor-parser-config'; styleUrls: ['sensor-parser-config-readonly.component.scss'] }) export class SensorParserConfigReadonlyComponent implements OnInit { - selectedSensorName: string; startStopInProgress: boolean = false; kafkaTopic: KafkaTopic = new KafkaTopic(); @@ -47,78 +46,149 @@ export class SensorParserConfigReadonlyComponent implements OnInit { sensorParserConfig: SensorParserConfig = new SensorParserConfig(); topologyStatus: TopologyStatus = new TopologyStatus(); sensorEnrichmentConfig: SensorEnrichmentConfig = new SensorEnrichmentConfig(); - grokStatement: string = ''; + grokStatement = {}; transformsConfigKeys: string[] = []; transformsConfigMap: {} = {}; rules: RiskLevelRule[] = []; transformLinkText = 'show more'; threatTriageLinkText = 'show more'; - editViewMetaData: {label?: string, value?: string, type?: string, model?: string, boldTitle?: boolean}[] = [ -{type: 'SEPARATOR', model: '', value: ''}, -{label: 'PARSER', model: 'sensorParserConfigHistory', value: 'parserClassName'}, -{label: 'LAST UPDATED', model: 'sensorParserConfigHistory', value: 'modifiedByDate'}, -{label: 'LAST EDITOR', model: 'sensorParserConfigHistory', value: 'modifiedBy'}, -{label: 'STATE', model: 'topologyStatus', value: 'sensorStatus'}, -{label: 'ORIGINATOR', model:
[28/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (cstella via mmiklavc)
METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (cstella via mmiklavc) Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e7e19fbb Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e7e19fbb Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e7e19fbb Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: e7e19fbb6491fa47d3794aebdac0280164afeb29 Parents: 5bfc08c Author: cstella Authored: Mon Oct 8 18:06:52 2018 -0600 Committer: Michael Miklavcic Committed: Thu Nov 15 16:51:13 2018 -0700 -- dependencies_with_url.csv | 33 ++-- .../METRON/CURRENT/configuration/metron-env.xml | 9 -- .../CURRENT/package/scripts/metron_service.py | 2 - .../package/scripts/params/params_linux.py | 3 +- .../METRON/CURRENT/themes/metron_theme.json | 10 -- .../rest/service/impl/MetaAlertServiceImpl.java | 2 +- metron-platform/elasticsearch-shaded/pom.xml| 28 +++- .../META-INF/log4j-provider.properties | 18 --- metron-platform/metron-elasticsearch/pom.xml| 29 +++- .../dao/ElasticsearchColumnMetadataDao.java | 82 +- .../elasticsearch/dao/ElasticsearchDao.java | 17 +- .../dao/ElasticsearchMetaAlertDao.java | 2 +- .../dao/ElasticsearchMetaAlertSearchDao.java| 6 +- .../dao/ElasticsearchMetaAlertUpdateDao.java| 4 +- .../dao/ElasticsearchRequestSubmitter.java | 13 +- .../dao/ElasticsearchRetrieveLatestDao.java | 27 ++-- .../dao/ElasticsearchSearchDao.java | 7 +- .../dao/ElasticsearchUpdateDao.java | 18 ++- .../utils/ElasticsearchClient.java | 156 +++ .../elasticsearch/utils/ElasticsearchUtils.java | 95 --- .../elasticsearch/utils/FieldMapping.java | 29 .../elasticsearch/utils/FieldProperties.java| 33 .../writer/ElasticsearchWriter.java | 22 +-- .../dao/ElasticsearchColumnMetadataDaoTest.java | 50 +++--- .../elasticsearch/dao/ElasticsearchDaoTest.java | 7 +- .../dao/ElasticsearchRequestSubmitterTest.java | 20 ++- .../ElasticsearchMetaAlertIntegrationTest.java | 9 +- .../ElasticsearchSearchIntegrationTest.java | 15 +- .../ElasticsearchUpdateIntegrationTest.java | 2 +- .../components/ElasticSearchComponent.java | 6 +- .../dao/metaalert/MetaAlertSearchDao.java | 4 +- .../dao/metaalert/MetaAlertIntegrationTest.java | 2 +- .../src/main/config/zookeeper/global.json | 2 +- pom.xml | 2 +- 34 files changed, 532 insertions(+), 232 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/e7e19fbb/dependencies_with_url.csv -- diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index 53977f3..66497c3 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -256,12 +256,8 @@ io.dropwizard.metrics:metrics-json:jar:3.1.5:compile,ASLv2,https://github.com/dr io.dropwizard.metrics:metrics-jvm:jar:3.1.5:compile,ASLv2,https://github.com/dropwizard/metrics io.netty:netty-all:jar:4.0.23.Final:compile,ASLv2, io.netty:netty-all:jar:4.0.23.Final:provided,ASLv2, -<<< HEAD io.netty:netty-all:jar:4.1.17.Final:compile,ASLv2, -=== io.netty:netty-all:jar:4.1.23.Final:compile,ASLv2, -io.netty:netty:jar:3.10.5.Final:compile,Apache License, Version 2.0,http://netty.io/ ->>> apache/master io.netty:netty:jar:3.6.2.Final:compile,Apache License, Version 2.0,http://netty.io/ io.netty:netty:jar:3.7.0.Final:compile,Apache License, Version 2.0,http://netty.io/ io.netty:netty:jar:3.9.9.Final:compile,Apache License, Version 2.0,http://netty.io/ @@ -472,20 +468,21 @@ org.eclipse.persistence:org.eclipse.persistence.jpa:jar:2.6.4:compile,EPL 1.0,ht com.github.ben-manes.caffeine:caffeine:jar:2.6.2:compile,ASLv2,https://github.com/ben-manes/caffeine/blob/v2.6.2/LICENSE com.google.code.gson:gson:jar:2.2:compile,ASLv2,https://github.com/google/gson com.google.code.gson:gson:jar:2.8.2:compile,ASLv2,https://github.com/google/gson - org.codehaus.plexus:plexus-classworlds:jar:2.4:compile - org.codehaus.plexus:plexus-component-annotations:jar:1.5.5:compile - org.codehaus.plexus:plexus-interpolation:jar:1.14:compile - org.codehaus.plexus:plexus-utils:jar:2.0.7:compile - org.jsoup:jsoup:jar:1.6.1:compile - org.sonatype.aether:aether-api:jar:1.12:compile - org.sonatype.aether:aether-connector-file:jar:1.12:compile - org.sonatype.aether:aether-connector-wagon:jar:1.12:compile - org.sonatype.aether:aether-impl:jar:1.12:compile - org.sonatype.aether:aether-spi:jar:1.12:compile - org.sonatype.aether:aether-util:jar:1.12:compile -
[23/52] [abbrv] metron git commit: METRON-1749 Update Angular to latest release in Management UI (sardell via nickwallen) closes apache/metron#1217
http://git-wip-us.apache.org/repos/asf/metron/blob/0c4c622b/metron-interface/metron-config/src/app/sensors/sensor-field-schema/sensor-field-schema.component.spec.ts -- diff --git a/metron-interface/metron-config/src/app/sensors/sensor-field-schema/sensor-field-schema.component.spec.ts b/metron-interface/metron-config/src/app/sensors/sensor-field-schema/sensor-field-schema.component.spec.ts index d2066ea..2cb44e2 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-field-schema/sensor-field-schema.component.spec.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-field-schema/sensor-field-schema.component.spec.ts @@ -19,505 +19,641 @@ /* tslint:disable:max-line-length */ import { TestBed, async, ComponentFixture } from '@angular/core/testing'; -import {Http} from '@angular/http'; -import {SimpleChanges, SimpleChange} from '@angular/core'; -import {SensorParserConfigService} from '../../service/sensor-parser-config.service'; -import {StellarService} from '../../service/stellar.service'; -import {MetronAlerts} from '../../shared/metron-alerts'; -import {SensorFieldSchemaModule} from './sensor-field-schema.module'; -import {SensorFieldSchemaComponent, FieldSchemaRow} from './sensor-field-schema.component'; -import {KafkaService} from '../../service/kafka.service'; -import {Observable} from 'rxjs/Observable'; -import {StellarFunctionDescription} from '../../model/stellar-function-description'; -import {SensorParserConfig} from '../../model/sensor-parser-config'; -import {SensorEnrichmentConfig, EnrichmentConfig, ThreatIntelConfig} from '../../model/sensor-enrichment-config'; -import {ParseMessageRequest} from '../../model/parse-message-request'; -import {AutocompleteOption} from '../../model/autocomplete-option'; -import {FieldTransformer} from '../../model/field-transformer'; -import {SensorEnrichmentConfigService} from '../../service/sensor-enrichment-config.service'; - +import { HttpClient } from '@angular/common/http'; +import { SimpleChanges, SimpleChange } from '@angular/core'; +import { SensorParserConfigService } from '../../service/sensor-parser-config.service'; +import { StellarService } from '../../service/stellar.service'; +import { MetronAlerts } from '../../shared/metron-alerts'; +import { SensorFieldSchemaModule } from './sensor-field-schema.module'; +import { + SensorFieldSchemaComponent, + FieldSchemaRow +} from './sensor-field-schema.component'; +import { KafkaService } from '../../service/kafka.service'; +import { Observable, throwError } from 'rxjs'; +import { StellarFunctionDescription } from '../../model/stellar-function-description'; +import { SensorParserConfig } from '../../model/sensor-parser-config'; +import { + SensorEnrichmentConfig, + EnrichmentConfig, + ThreatIntelConfig +} from '../../model/sensor-enrichment-config'; +import { ParseMessageRequest } from '../../model/parse-message-request'; +import { AutocompleteOption } from '../../model/autocomplete-option'; +import { FieldTransformer } from '../../model/field-transformer'; +import { SensorEnrichmentConfigService } from '../../service/sensor-enrichment-config.service'; class MockSensorParserConfigService { - -parseMessage(parseMessageRequest: ParseMessageRequest): Observable<{}> { -let parsedJson = { -'elapsed': 415, -'code': 200, -'ip_dst_addr': '207.109.73.154', -'original_string': '1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/', -'method': 'GET', -'bytes': 337891, -'action': 'TCP_MISS', -'ip_src_addr': '127.0.0.1', -'url': 'http://www.aliexpress.com/af/shoes.html?', -'timestamp': '1467011157.401' -}; -return Observable.create((observable) => { -observable.next(parsedJson); -observable.complete(); -}); -} + parseMessage(parseMessageRequest: ParseMessageRequest): Observable<{}> { +let parsedJson = { + elapsed: 415, + code: 200, + ip_dst_addr: '207.109.73.154', + original_string: +'1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/', + method: 'GET', + bytes: 337891, + action: 'TCP_MISS', + ip_src_addr: '127.0.0.1', + url: 'http://www.aliexpress.com/af/shoes.html?', + timestamp: '1467011157.401' +}; +return Observable.create(observable => { +
[22/52] [abbrv] metron git commit: METRON-1749 Update Angular to latest release in Management UI (sardell via nickwallen) closes apache/metron#1217
http://git-wip-us.apache.org/repos/asf/metron/blob/0c4c622b/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts -- diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts b/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts index 7d41003..2904ef5 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts @@ -15,23 +15,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import {Component, OnInit} from '@angular/core'; -import {KafkaService} from '../../service/kafka.service'; -import {Router, ActivatedRoute} from '@angular/router'; -import {KafkaTopic} from '../../model/kafka-topic'; -import {MetronAlerts} from '../../shared/metron-alerts'; -import {SensorParserConfigService} from '../../service/sensor-parser-config.service'; -import {StormService} from '../../service/storm.service'; -import {TopologyStatus} from '../../model/topology-status'; -import {SensorParserConfigHistoryService} from '../../service/sensor-parser-config-history.service'; -import {SensorParserConfigHistory} from '../../model/sensor-parser-config-history'; -import {SensorEnrichmentConfigService} from '../../service/sensor-enrichment-config.service'; -import {SensorEnrichmentConfig} from '../../model/sensor-enrichment-config'; -import {RiskLevelRule} from '../../model/risk-level-rule'; -import {HdfsService} from '../../service/hdfs.service'; -import {RestError} from '../../model/rest-error'; -import {GrokValidationService} from '../../service/grok-validation.service'; -import {SensorParserConfig} from '../../model/sensor-parser-config'; +import { Component, OnInit } from '@angular/core'; +import { KafkaService } from '../../service/kafka.service'; +import { Router, ActivatedRoute } from '@angular/router'; +import { KafkaTopic } from '../../model/kafka-topic'; +import { MetronAlerts } from '../../shared/metron-alerts'; +import { SensorParserConfigService } from '../../service/sensor-parser-config.service'; +import { StormService } from '../../service/storm.service'; +import { TopologyStatus } from '../../model/topology-status'; +import { SensorParserConfigHistoryService } from '../../service/sensor-parser-config-history.service'; +import { SensorParserConfigHistory } from '../../model/sensor-parser-config-history'; +import { SensorEnrichmentConfigService } from '../../service/sensor-enrichment-config.service'; +import { SensorEnrichmentConfig } from '../../model/sensor-enrichment-config'; +import { RiskLevelRule } from '../../model/risk-level-rule'; +import { HdfsService } from '../../service/hdfs.service'; +import { RestError } from '../../model/rest-error'; +import { GrokValidationService } from '../../service/grok-validation.service'; +import { SensorParserConfig } from '../../model/sensor-parser-config'; @Component({ selector: 'metron-config-sensor-parser-readonly', @@ -39,7 +39,6 @@ import {SensorParserConfig} from '../../model/sensor-parser-config'; styleUrls: ['sensor-parser-config-readonly.component.scss'] }) export class SensorParserConfigReadonlyComponent implements OnInit { - selectedSensorName: string; startStopInProgress: boolean = false; kafkaTopic: KafkaTopic = new KafkaTopic(); @@ -47,78 +46,149 @@ export class SensorParserConfigReadonlyComponent implements OnInit { sensorParserConfig: SensorParserConfig = new SensorParserConfig(); topologyStatus: TopologyStatus = new TopologyStatus(); sensorEnrichmentConfig: SensorEnrichmentConfig = new SensorEnrichmentConfig(); - grokStatement: string = ''; + grokStatement = {}; transformsConfigKeys: string[] = []; transformsConfigMap: {} = {}; rules: RiskLevelRule[] = []; transformLinkText = 'show more'; threatTriageLinkText = 'show more'; - editViewMetaData: {label?: string, value?: string, type?: string, model?: string, boldTitle?: boolean}[] = [ -{type: 'SEPARATOR', model: '', value: ''}, -{label: 'PARSER', model: 'sensorParserConfigHistory', value: 'parserClassName'}, -{label: 'LAST UPDATED', model: 'sensorParserConfigHistory', value: 'modifiedByDate'}, -{label: 'LAST EDITOR', model: 'sensorParserConfigHistory', value: 'modifiedBy'}, -{label: 'STATE', model: 'topologyStatus', value: 'sensorStatus'}, -{label: 'ORIGINATOR', model:
[09/52] [abbrv] metron git commit: METRON-1853: Add shutdown hook to Stellar BaseFunctionResolver (mmiklavc via mmiklavc) closes apache/metron#1251
METRON-1853: Add shutdown hook to Stellar BaseFunctionResolver (mmiklavc via
mmiklavc) closes apache/metron#1251
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/85cd21aa
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/85cd21aa
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/85cd21aa
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 85cd21aa0f5045184c168248dc2b81c1cfd41ddd
Parents: b9461e7
Author: mmiklavc
Authored: Tue Nov 6 18:09:56 2018 -0700
Committer: Michael Miklavcic
Committed: Tue Nov 6 18:09:56 2018 -0700
--
.../ElasticsearchSearchIntegrationTest.java | 1 -
.../metron/stellar/dsl/StellarFunction.java | 9 +-
.../metron/stellar/dsl/StellarFunctions.java| 5 +
.../resolver/BaseFunctionResolver.java | 44 +
.../functions/resolver/FunctionResolver.java| 14 +-
.../stellar/dsl/functions/BasicStellarTest.java | 20 ++-
.../resolver/BaseFunctionResolverTest.java | 169 +++
7 files changed, 251 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 1d2d48e..8187468 100644
---
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -25,7 +25,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.Constants;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
index efdd185..4fabfaf 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunction.java
@@ -17,10 +17,17 @@
*/
package org.apache.metron.stellar.dsl;
+import java.io.Closeable;
+import java.io.IOException;
import java.util.List;
-public interface StellarFunction {
+public interface StellarFunction extends Closeable {
Object apply(List args, Context context) throws ParseException;
void initialize(Context context);
boolean isInitialized();
+
+ @Override
+ default void close() throws IOException {
+
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
index dfec90e..73df82f 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/StellarFunctions.java
@@ -18,6 +18,7 @@
package org.apache.metron.stellar.dsl;
+import java.io.IOException;
import org.apache.metron.stellar.dsl.functions.resolver.FunctionResolver;
import
org.apache.metron.stellar.dsl.functions.resolver.SingletonFunctionResolver;
@@ -30,4 +31,8 @@ public class StellarFunctions {
public static void initialize(Context context) {
SingletonFunctionResolver.getInstance().initialize(context);
}
+
+ public static void close() throws IOException {
+SingletonFunctionResolver.getInstance().close();
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/85cd21aa/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/resolver/BaseFunctionResolver.java
[07/52] [abbrv] metron git commit: METRON-1855: Make unified enrichment topology the default and deprecate split-join (mmiklavc via mmiklavc) closes apache/metron#1252
METRON-1855: Make unified enrichment topology the default and deprecate
split-join (mmiklavc via mmiklavc) closes apache/metron#1252
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bf6b07f7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bf6b07f7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bf6b07f7
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: bf6b07f7cbea3d210878554c7ce7a1bc091b59ee
Parents: fdfca3b
Author: mmiklavc
Authored: Mon Nov 5 16:30:43 2018 -0700
Committer: Michael Miklavcic
Committed: Mon Nov 5 16:30:43 2018 -0700
--
Upgrading.md| 17
.../configuration/metron-enrichment-env.xml | 8 ++--
.../METRON/CURRENT/themes/metron_theme.json | 12 +++---
metron-platform/Performance-tuning-guide.md | 6 ++-
metron-platform/metron-enrichment/README.md | 43 +---
.../main/scripts/start_enrichment_topology.sh | 4 +-
6 files changed, 54 insertions(+), 36 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/bf6b07f7/Upgrading.md
--
diff --git a/Upgrading.md b/Upgrading.md
index 2124ac5..a0dd5d3 100644
--- a/Upgrading.md
+++ b/Upgrading.md
@@ -19,6 +19,23 @@ limitations under the License.
This document constitutes a per-version listing of changes of
configuration which are non-backwards compatible.
+## 0.6.0 to 0.6.1
+
+### [METRON-1855: Make unified enrichment topology the default and deprecate
split-join](https://issues.apache.org/jira/browse/METRON-1855)
+The unified enrichment topology will be the new default in this release,
+and the split-join enrichment topology is now considered deprecated.
+If you wish to keep the deprecated split-join enrichment topology,
+you will need to make the following changes:
+
+* In Ambari > Metron > Config > Enrichment set the enrichment_topology setting
to "Split-Join"
+* If running `start_enrichment_topology.sh` manually, pass in the parameters
to start the Split-Join topology as follows
+
+```
+$METRON_HOME/bin/start_enrichment_topology.sh --remote
$METRON_HOME/flux/enrichment/remote-splitjoin.yaml --filter
$METRON_HOME/config/enrichment-splitjoin.properties
+```
+
+* Restart the enrichment topology
+
## 0.4.2 to 0.5.0
### [METRON-941: native PaloAlto parser corrupts message when having a comma
in the payload](https://issues.apache.org/jira/browse/METRON-941)
http://git-wip-us.apache.org/repos/asf/metron/blob/bf6b07f7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
index b41c455..69dce3f 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
@@ -165,17 +165,17 @@
enrichment_topology
-Which Enrichment topology to execute
-Split-Join
+Which Enrichment topology to execute. Note: Split-Join is
deprecated in favor of the Unified topology.
+Unified
Enrichment Topology
value-list
- Split-Join
+ Unified
- Unified
+ Split-Join
1
http://git-wip-us.apache.org/repos/asf/metron/blob/bf6b07f7/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 1d7b6c5..46c06dd 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -125,7 +125,7 @@
]
},
{
- "name": "section-enrichment-splitjoin",
+ "name":
[34/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClientFactory.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClientFactory.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClientFactory.java
new file mode 100644
index 000..4e0b2fe
--- /dev/null
+++
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClientFactory.java
@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.elasticsearch.client;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import javax.net.ssl.SSLContext;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.nio.reactor.IOReactorConfig;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.http.ssl.SSLContexts;
+import org.apache.metron.elasticsearch.config.ElasticsearchClientConfig;
+import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
+import org.apache.metron.elasticsearch.utils.ElasticsearchUtils.HostnamePort;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Main entry point to create the ES client.
+ */
+public class ElasticsearchClientFactory {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final String ES_SETTINGS_KEY = "es.client.settings"; // es
config key in global config
+
+ /**
+ * Creates an Elasticsearch client from settings provided via the global
config.
+ *
+ * @return new client
+ */
+ public static ElasticsearchClient create(Map globalConfig) {
+ElasticsearchClientConfig esClientConfig = new ElasticsearchClientConfig(
+getEsSettings(globalConfig));
+HttpHost[] httpHosts = getHttpHosts(globalConfig,
esClientConfig.getConnectionScheme());
+RestClientBuilder builder = RestClient.builder(httpHosts);
+
+builder.setRequestConfigCallback(reqConfigBuilder -> {
+ // Modifies request config builder with connection and socket timeouts.
+ //
https://www.elastic.co/guide/en/elasticsearch/client/java-rest/5.6/_timeouts.html
+
reqConfigBuilder.setConnectTimeout(esClientConfig.getConnectTimeoutMillis());
+
reqConfigBuilder.setSocketTimeout(esClientConfig.getSocketTimeoutMillis());
+ return reqConfigBuilder;
+});
+
builder.setMaxRetryTimeoutMillis(esClientConfig.getMaxRetryTimeoutMillis());
+
+builder.setHttpClientConfigCallback(clientBuilder -> {
+
clientBuilder.setDefaultIOReactorConfig(getIOReactorConfig(esClientConfig));
+
clientBuilder.setDefaultCredentialsProvider(getCredentialsProvider(esClientConfig));
+ clientBuilder.setSSLContext(getSSLContext(esClientConfig));
+ return clientBuilder;
+});
+
+RestClient lowLevelClient = builder.build();
+RestHighLevelClient client = new RestHighLevelClient(lowLevelClient);
+return new ElasticsearchClient(lowLevelClient, client);
+ }
+
+ private static Map getEsSettings(Map
globalConfig) {
+return (Map) globalConfig.getOrDefault(ES_SETTINGS_KEY,
new HashMap<>());
+ }
+
+ private static HttpHost[] getHttpHosts(Map
[32/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
--
diff --git
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
index 489eb00..1cf9fb7 100644
---
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
+++
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,18 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.metron.parsers.fireeye;
import com.google.common.base.Joiner;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.metron.parsers.utils.ParserUtils;
-import org.apache.metron.parsers.BasicParser;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -34,189 +30,162 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.parsers.BasicParser;
+import org.apache.metron.parsers.utils.ParserUtils;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public class BasicFireEyeParser extends BasicParser {
-
- private static final long serialVersionUID = 6328907550159134550L;
- protected static final Logger LOG = LoggerFactory
- .getLogger(BasicFireEyeParser.class);
-
-
- String tsRegex
="([a-zA-Z]{3})\\s+(\\d+)\\s+(\\d+\\:\\d+\\:\\d+)\\s+(\\d+\\.\\d+\\.\\d+\\.\\d+)";
-
-
- Pattern tsPattern = Pattern.compile(tsRegex);
- // private transient static MetronGrok grok;
- // private transient static InputStream pattern_url;
-
- public BasicFireEyeParser() throws Exception {
- // pattern_url =
getClass().getClassLoader().getResourceAsStream(
- // "patterns/fireeye");
- //
- // File file = ParserUtils.stream2file(pattern_url);
- // grok = MetronGrok.create(file.getPath());
- //
- // grok.compile("%{FIREEYE_BASE}");
- }
-
- @Override
- public void configure(Map parserConfig) {
-
- }
-
- @Override
- public void init() {
-
- }
-
- @Override
- public List parse(byte[] raw_message) {
- String toParse = "";
- List messages = new ArrayList<>();
- try {
-
- toParse = new String(raw_message, "UTF-8");
-
- // String[] mTokens = toParse.split(" ");
-
- String positveIntPattern = "<[1-9][0-9]*>";
- Pattern p = Pattern.compile(positveIntPattern);
- Matcher m = p.matcher(toParse);
-
- String delimiter = "";
-
- while (m.find()) {
- delimiter = m.group();
-
- }
-
- if (!StringUtils.isBlank(delimiter)) {
- String[] tokens = toParse.split(delimiter);
-
- if (tokens.length > 1)
- toParse = delimiter + tokens[1];
-
- }
-
- JSONObject toReturn = parseMessage(toParse);
-
- toReturn.put("timestamp",
getTimeStamp(toParse,delimiter));
- messages.add(toReturn);
- return messages;
-
- } catch (Exception e) {
- e.printStackTrace();
- return null;
- }
-
- }
-
- private long getTimeStamp(String toParse,String delimiter) throws
ParseException {
-
- long ts = 0;
- String month = null;
- String day = null;
- String time = null;
- Matcher tsMatcher = tsPattern.matcher(toParse);
- if (tsMatcher.find()) {
- month = tsMatcher.group(1);
- day = tsMatcher.group(2);
-
[01/52] [abbrv] metron git commit: METRON-1803: Integrate Cypress with Travis (tiborm via mmiklavc) closes apache/metron#1226
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1090-stellar-assignment 3df949877 -> f548b191b
http://git-wip-us.apache.org/repos/asf/metron/blob/9b6260fd/metron-interface/metron-alerts/cypress/integration/pcap/pcap.spec.js
--
diff --git
a/metron-interface/metron-alerts/cypress/integration/pcap/pcap.spec.js
b/metron-interface/metron-alerts/cypress/integration/pcap/pcap.spec.js
new file mode 100644
index 000..58f7d26
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress/integration/pcap/pcap.spec.js
@@ -0,0 +1,228 @@
+///
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+context('PCAP Tab', () => {
+
+ beforeEach(() => {
+cy.server();
+cy.route({
+ method: 'GET',
+ url: '/api/v1/user',
+ response: 'user'
+});
+
+cy.route('GET', 'config', 'fixture:config.json');
+cy.route('POST', 'search', 'fixture:search.json');
+
+cy.route({
+ method: 'GET',
+ url: '/api/v1/pcap?state=*',
+ response: []
+}).as('runningJobs');
+
+cy.visit('http://localhost:4200/login');
+cy.get('[name="user"]').type('user');
+cy.get('[name="password"]').type('password');
+cy.contains('LOG IN').click();
+ });
+
+ afterEach(() => {
+cy.get('.logout-link').click();
+ });
+
+ it('checking running jobs on navigating to PCAP tab', () => {
+cy.contains('PCAP').click();
+cy.wait('@runningJobs').its('url').should('include', '?state=RUNNING');
+ });
+
+ it('submitting PCAP job request', () => {
+cy.contains('PCAP').click();
+cy.route('POST', '/api/v1/pcap/fixed', 'fixture:pcap.status-00.json')
+ .as('postingPcapJob');
+
+cy.get('[data-qe-id="ip-src-addr"]').type('222.123.111.000');
+cy.get('[data-qe-id="ip-dst-addr"]').type('111.123.222.000');
+cy.get('[data-qe-id="ip-src-port"]').type('');
+cy.get('[data-qe-id="ip-dst-port"]').type('');
+cy.get('[data-qe-id="protocol"]').type('24');
+cy.get('[data-qe-id="include-reverse"]').check();
+cy.get('[data-qe-id="packet-filter"]').type('filter');
+
+cy.get('[data-qe-id="submit-button"]').click();
+
+cy.wait('@postingPcapJob').then((xhr) => {
+ expect(xhr.request.body.ipSrcAddr).to.equal('222.123.111.000');
+ expect(xhr.request.body.ipDstAddr).to.equal('111.123.222.000');
+ expect(xhr.request.body.ipSrcPort).to.equal('');
+ expect(xhr.request.body.ipDstPort).to.equal('');
+ expect(xhr.request.body.protocol).to.equal('24');
+ expect(xhr.request.body.includeReverse).to.equal(true);
+ expect(xhr.request.body.packetFilter).to.equal('filter');
+});
+ });
+
+ it('requesting job status', () => {
+cy.route('POST', '/api/v1/pcap/fixed', 'fixture:pcap.status-00.json');
+cy.route('GET', '/api/v1/pcap/*',
'fixture:pcap.status-01.json').as('jobStatusCheck');
+
+cy.contains('PCAP').click();
+cy.get('[data-qe-id="submit-button"]').click();
+
+cy.wait('@jobStatusCheck').its('url').should('include',
'/api/v1/pcap/job_1537878471649_0001');
+ });
+
+ it('process status in percentage', () => {
+cy.route('POST', '/api/v1/pcap/fixed', 'fixture:pcap.status-00.json');
+cy.route('GET', '/api/v1/pcap/*',
'fixture:pcap.status-01.json').as('jobStatusCheck');
+
+cy.contains('PCAP').click();
+cy.get('[data-qe-id="submit-button"]').click();
+
+cy.wait('@jobStatusCheck');
+
+cy.contains('75%').should('be.visible');
+ });
+
+ it('getting pcap json', () => {
+cy.route('POST', '/api/v1/pcap/fixed', 'fixture:pcap.status-00.json');
+cy.route('GET', '/api/v1/pcap/*',
'fixture:pcap.status-02.json').as('statusCheck');
+cy.route('GET', '/api/v1/pcap/*/pdml*',
'fixture:pcap.page-01.json').as('gettingPdml');
+
+cy.contains('PCAP').click();
+cy.get('[data-qe-id="submit-button"]').click();
+
+cy.wait('@statusCheck');
+
+cy.wait('@gettingPdml').its('url').should('include',
'/api/v1/pcap/job_1537878471649_0001/pdml?page=1');
+ });
+
+
+ it('rendering pcap table', () => {
+cy.route('POST', '/api/v1/pcap/fixed', 'fixture:pcap.status-00.json');
+cy.route('GET', '/api/v1/pcap/*',
[48/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
--
diff --git
a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
index c33644f..83800af 100644
---
a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
+++
b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
@@ -58,8 +58,11 @@ import static
org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INP
import static
org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_END;
import static
org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
import static
org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+import static
org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_READER;
import static org.junit.Assert.assertTrue;
+import static org.apache.metron.profiler.spark.reader.TelemetryReaders.*;
+
/**
* An integration test for the {@link BatchProfiler}.
*/
@@ -159,8 +162,8 @@ public class BatchProfilerIntegrationTest {
@Test
public void testBatchProfilerWithJSON() throws Exception {
// the input telemetry is text/json stored in the local filesystem
+profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), JSON.toString());
profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(),
"src/test/resources/telemetry.json");
-profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "text");
BatchProfiler profiler = new BatchProfiler();
profiler.run(spark, profilerProperties, getGlobals(), readerProperties,
getProfile());
@@ -170,20 +173,41 @@ public class BatchProfilerIntegrationTest {
@Test
public void testBatchProfilerWithORC() throws Exception {
-// re-write the test data as ORC
+// re-write the test data as column-oriented ORC
String pathToORC = tempFolder.getRoot().getAbsolutePath();
spark.read()
-.format("text")
+.format("json")
.load("src/test/resources/telemetry.json")
-.as(Encoders.STRING())
.write()
.mode("overwrite")
.format("org.apache.spark.sql.execution.datasources.orc")
.save(pathToORC);
// tell the profiler to use the ORC input data
+profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), ORC.toString());
profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToORC);
-profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(),
"org.apache.spark.sql.execution.datasources.orc");
+
+BatchProfiler profiler = new BatchProfiler();
+profiler.run(spark, profilerProperties, getGlobals(), readerProperties,
getProfile());
+
+validateProfiles();
+ }
+
+ @Test
+ public void testBatchProfilerWithParquet() throws Exception {
+// re-write the test data as column-oriented ORC
+String inputPath = tempFolder.getRoot().getAbsolutePath();
+spark.read()
+.format("json")
+.load("src/test/resources/telemetry.json")
+.write()
+.mode("overwrite")
+.format("parquet")
+.save(inputPath);
+
+// tell the profiler to use the ORC input data
+profilerProperties.put(TELEMETRY_INPUT_READER.getKey(),
PARQUET.toString());
+profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), inputPath);
BatchProfiler profiler = new BatchProfiler();
profiler.run(spark, profilerProperties, getGlobals(), readerProperties,
getProfile());
@@ -206,7 +230,9 @@ public class BatchProfilerIntegrationTest {
.save(pathToCSV);
// tell the profiler to use the CSV input data
+// CSV is an example of needing to define both the reader and the input
format
profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), pathToCSV);
+profilerProperties.put(TELEMETRY_INPUT_READER.getKey(), "text");
profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "csv");
// set a reader property; tell the reader to expect a header
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java
--
diff --git
a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/reader/ColumnEncodedTelemetryReaderTest.java
[20/52] [abbrv] metron git commit: METRON-1749 Update Angular to latest release in Management UI (sardell via nickwallen) closes apache/metron#1217
http://git-wip-us.apache.org/repos/asf/metron/blob/0c4c622b/metron-interface/metron-config/src/app/sensors/sensor-storm-settings/sensor-storm-settings.component.spec.ts
--
diff --git
a/metron-interface/metron-config/src/app/sensors/sensor-storm-settings/sensor-storm-settings.component.spec.ts
b/metron-interface/metron-config/src/app/sensors/sensor-storm-settings/sensor-storm-settings.component.spec.ts
index 02f1fd9..6549b4b 100644
---
a/metron-interface/metron-config/src/app/sensors/sensor-storm-settings/sensor-storm-settings.component.spec.ts
+++
b/metron-interface/metron-config/src/app/sensors/sensor-storm-settings/sensor-storm-settings.component.spec.ts
@@ -16,153 +16,161 @@
* limitations under the License.
*/
-import {async, TestBed, ComponentFixture} from '@angular/core/testing';
-import {SensorStormSettingsComponent} from './sensor-storm-settings.component';
-import {SharedModule} from '../../shared/shared.module';
-import {SimpleChanges, SimpleChange} from '@angular/core';
-import {SensorParserConfig} from '../../model/sensor-parser-config';
-import {SensorStormSettingsModule} from './sensor-storm-settings.module';
-import '../../rxjs-operators';
+import { async, TestBed, ComponentFixture } from '@angular/core/testing';
+import { SensorStormSettingsComponent } from
'./sensor-storm-settings.component';
+import { SharedModule } from '../../shared/shared.module';
+import { SimpleChanges, SimpleChange } from '@angular/core';
+import { SensorParserConfig } from '../../model/sensor-parser-config';
+import { SensorStormSettingsModule } from './sensor-storm-settings.module';
describe('Component: SensorStormSettingsComponent', () => {
-
-let fixture: ComponentFixture;
-let component: SensorStormSettingsComponent;
-let sensorParserConfig: SensorParserConfig = new SensorParserConfig();
-sensorParserConfig.sensorTopic = 'bro';
-sensorParserConfig.parserClassName =
'org.apache.metron.parsers.bro.BasicBroParser';
-sensorParserConfig.parserConfig = {};
-sensorParserConfig.numWorkers = 2;
-sensorParserConfig.numAckers = 2;
-sensorParserConfig.spoutParallelism = 2;
-sensorParserConfig.spoutNumTasks = 2;
-sensorParserConfig.parserParallelism = 2;
-sensorParserConfig.parserNumTasks = 2;
-sensorParserConfig.errorWriterParallelism = 2;
-sensorParserConfig.errorWriterNumTasks = 2;
-sensorParserConfig.spoutConfig = {'spoutConfigProp': 'spoutConfigValue1'};
-sensorParserConfig.stormConfig = {'stormConfigProp': 'stormConfigValue1'};
-
-beforeEach(async(() => {
-TestBed.configureTestingModule({
-imports: [SharedModule, SensorStormSettingsModule],
-});
-
-fixture = TestBed.createComponent(SensorStormSettingsComponent);
-component = fixture.componentInstance;
-}));
-
-it('should create an instance', () => {
-expect(component).toBeDefined();
-});
-
-it('should create an instance', () => {
-spyOn(component, 'init');
-let changes: SimpleChanges = {'showStormSettings': new
SimpleChange(false, true)};
-
-component.ngOnChanges(changes);
-expect(component.init).toHaveBeenCalled();
-
-changes = {'showStormSettings': new SimpleChange(true, false)};
-component.ngOnChanges(changes);
-expect(component.init['calls'].count()).toEqual(1);
-
-fixture.destroy();
+ let fixture: ComponentFixture;
+ let component: SensorStormSettingsComponent;
+ let sensorParserConfig: SensorParserConfig = new SensorParserConfig();
+ sensorParserConfig.sensorTopic = 'bro';
+ sensorParserConfig.parserClassName =
+'org.apache.metron.parsers.bro.BasicBroParser';
+ sensorParserConfig.parserConfig = {};
+ sensorParserConfig.numWorkers = 2;
+ sensorParserConfig.numAckers = 2;
+ sensorParserConfig.spoutParallelism = 2;
+ sensorParserConfig.spoutNumTasks = 2;
+ sensorParserConfig.parserParallelism = 2;
+ sensorParserConfig.parserNumTasks = 2;
+ sensorParserConfig.errorWriterParallelism = 2;
+ sensorParserConfig.errorWriterNumTasks = 2;
+ sensorParserConfig.spoutConfig = { spoutConfigProp: 'spoutConfigValue1' };
+ sensorParserConfig.stormConfig = { stormConfigProp: 'stormConfigValue1' };
+
+ beforeEach(async(() => {
+TestBed.configureTestingModule({
+ imports: [SharedModule, SensorStormSettingsModule]
});
-it('should initialise the fields', () => {
-
-component.init();
-expect(component.newSensorParserConfig).toEqual(new
SensorParserConfig());
-
-component.sensorParserConfig = sensorParserConfig;
-component.init();
-expect(component.newSensorParserConfig).toEqual(sensorParserConfig);
-expect(component.newSpoutConfig).toEqual('{\n\t"spoutConfigProp":
"spoutConfigValue1"\n}');
-expect(component.newStormConfig).toEqual('{\n\t"stormConfigProp":
"stormConfigValue1"\n}');
-
-
[06/52] [abbrv] metron git commit: METRON-1790 Unsubscribe from every observable in the pcap panel UI component (ruffle via nickwallen) closes apache/metron#1208
METRON-1790 Unsubscribe from every observable in the pcap panel UI component
(ruffle via nickwallen) closes apache/metron#1208
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fdfca3b2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fdfca3b2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fdfca3b2
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: fdfca3b2675d0dfcdf7960c5db789a8c9386b5db
Parents: 9b6260f
Author: ruffle
Authored: Mon Nov 5 10:45:33 2018 -0500
Committer: nickallen
Committed: Mon Nov 5 10:45:33 2018 -0500
--
.../app/pcap/pcap-panel/pcap-panel.component.ts | 68 +---
1 file changed, 32 insertions(+), 36 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/fdfca3b2/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
index f767020..fdd75f7 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
@@ -35,11 +35,6 @@ export class PcapPanelComponent implements OnInit, OnDestroy
{
pdml: Pdml = null;
pcapRequest: PcapRequest;
resetPaginationForSearch: boolean;
-
- statusSubscription: Subscription;
- cancelSubscription: Subscription;
- submitSubscription: Subscription;
- getSubscription: Subscription;
queryRunning = false;
queryId: string;
progressWidth = 0;
@@ -47,28 +42,33 @@ export class PcapPanelComponent implements OnInit,
OnDestroy {
savedPcapRequest: {};
errorMsg: string;
cancelConfirmMessage = 'Are you sure want to cancel the running query?';
+ subscriptions: {
+[key: string]: Subscription
+ } = {};
constructor(private pcapService: PcapService) { }
ngOnInit() {
this.pcapRequest = new PcapRequest();
-this.pcapService.getRunningJob().subscribe((statusResponses:
PcapStatusResponse[]) => {
+this.subscriptions['runningJobSubscription'] =
this.pcapService.getRunningJob().subscribe((statusResponses:
PcapStatusResponse[]) => {
if (statusResponses.length > 0) {
// Assume the first job in the list is the running job
this.queryRunning = true;
let statusResponse = statusResponses[0];
this.updateStatus(statusResponse);
this.startPolling(statusResponse.jobId);
-
this.pcapService.getPcapRequest(statusResponse.jobId).subscribe((pcapRequest:
PcapRequest) => {
- this.pcapRequest = pcapRequest;
-});
+this.subscriptions['pcapRequestSubscription'] =
this.pcapService.getPcapRequest(statusResponse.jobId).subscribe(
+ (pcapRequest: PcapRequest) => {
+this.pcapRequest = pcapRequest;
+ }
+);
}
});
}
changePage(page) {
this.pagination.selectedPage = page;
-this.pcapService.getPackets(this.queryId,
this.pagination.selectedPage).toPromise().then(pdml => {
+this.subscriptions['packetSubscription'] =
this.pcapService.getPackets(this.queryId,
this.pagination.selectedPage).subscribe(pdml => {
this.pdml = pdml;
});
}
@@ -81,26 +81,28 @@ export class PcapPanelComponent implements OnInit,
OnDestroy {
this.pdml = null;
this.progressWidth = 0;
this.errorMsg = null;
-this.submitSubscription =
this.pcapService.submitRequest(pcapRequest).subscribe((submitResponse:
PcapStatusResponse) => {
- let id = submitResponse.jobId;
- if (!id) {
-this.errorMsg = submitResponse.description;
-this.queryRunning = false;
- } else {
-this.startPolling(id);
+this.subscriptions['submitSubscription'] =
this.pcapService.submitRequest(pcapRequest).subscribe(
+ (submitResponse: PcapStatusResponse) => {
+let id = submitResponse.jobId;
+if (!id) {
+ this.errorMsg = submitResponse.description;
+ this.queryRunning = false;
+} else {
+ this.startPolling(id);
+}
+ }, (error: any) => {
+this.errorMsg = `Response message: ${error.message}. Something went
wrong with your query submission!`;
}
-}, (error: any) => {
- this.errorMsg = `Response message: ${error.message}. Something went
wrong with your query submission!`;
-});
+);
}
startPolling(id: string) {
this.queryId = id;
this.errorMsg = null;
-this.statusSubscription =
this.pcapService.pollStatus(id).subscribe((statusResponse: PcapStatusResponse)
=> {
+this.subscriptions['statusSubscription'] =
[25/52] [abbrv] metron git commit: METRON-1749 Update Angular to latest release in Management UI (sardell via nickwallen) closes apache/metron#1217
http://git-wip-us.apache.org/repos/asf/metron/blob/0c4c622b/metron-interface/metron-config/package-lock.json
--
diff --git a/metron-interface/metron-config/package-lock.json
b/metron-interface/metron-config/package-lock.json
index fff8cb7..f2a264d 100644
--- a/metron-interface/metron-config/package-lock.json
+++ b/metron-interface/metron-config/package-lock.json
@@ -4,180 +4,530 @@
"lockfileVersion": 1,
"requires": true,
"dependencies": {
-"@angular-cli/ast-tools": {
- "version": "1.0.16",
- "resolved":
"https://registry.npmjs.org/@angular-cli/ast-tools/-/ast-tools-1.0.16.tgz;,
- "integrity": "sha1-YxmULBol+4TjKUID6fejJmMvzlA=",
+"@angular-devkit/architect": {
+ "version": "0.8.3",
+ "resolved":
"https://registry.npmjs.org/@angular-devkit/architect/-/architect-0.8.3.tgz;,
+ "integrity":
"sha512-cFku50grgEJPg1CZZ0DXt4CkA6WnV6zN3hCXzpWbOfc/Id923Mml/jsEaoByeXHsRqb5rIZKZAhz7R509ya8OQ==",
"dev": true,
"requires": {
-"@angular/tsc-wrapped": "0.5.2",
-"denodeify": "1.2.1",
-"rxjs": "5.1.0",
-"typescript": "2.0.10"
+"@angular-devkit/core": "0.8.3",
+"rxjs": "6.2.2"
},
"dependencies": {
-"@angular/tsc-wrapped": {
- "version": "0.5.2",
- "resolved":
"https://registry.npmjs.org/@angular/tsc-wrapped/-/tsc-wrapped-0.5.2.tgz;,
- "integrity": "sha1-Lt30csRn/LM06pTe3aqnGZDFpII=",
+"@angular-devkit/core": {
+ "version": "0.8.3",
+ "resolved":
"https://registry.npmjs.org/@angular-devkit/core/-/core-0.8.3.tgz;,
+ "integrity":
"sha512-2KHt5osMs3zACYXev20ZU5SXdWoinoKwZkj2caj2LCj9W7QNHmsz34QvaygNq7YdJzF3jkXkdy0GSUgUgDke0w==",
"dev": true,
"requires": {
-"tsickle": "0.2.5"
+"ajv": "6.4.0",
+"chokidar": "2.0.4",
+"rxjs": "6.2.2",
+"source-map": "0.5.7"
}
-},
-"denodeify": {
- "version": "1.2.1",
- "resolved":
"https://registry.npmjs.org/denodeify/-/denodeify-1.2.1.tgz;,
- "integrity": "sha1-OjYof1A05pnnV3kBBSwubJQlFjE=",
+}
+ }
+},
+"@angular-devkit/build-angular": {
+ "version": "0.8.3",
+ "resolved":
"https://registry.npmjs.org/@angular-devkit/build-angular/-/build-angular-0.8.3.tgz;,
+ "integrity":
"sha512-NWwWV+6apvCGmllWjwwy9Pmj5uK5tVGL/xIVQgSGC5waLmW/vFWNRXCI50ji5UPP+vAeRi/pWdXWMxuoVA08FA==",
+ "dev": true,
+ "requires": {
+"@angular-devkit/architect": "0.8.3",
+"@angular-devkit/build-optimizer": "0.8.3",
+"@angular-devkit/build-webpack": "0.8.3",
+"@angular-devkit/core": "0.8.3",
+"@ngtools/webpack": "6.2.3",
+"ajv": "6.4.0",
+"autoprefixer": "8.6.5",
+"circular-dependency-plugin": "5.0.2",
+"clean-css": "4.2.1",
+"copy-webpack-plugin": "4.5.2",
+"file-loader": "1.1.11",
+"glob": "7.1.3",
+"html-webpack-plugin": "3.2.0",
+"istanbul": "0.4.5",
+"istanbul-instrumenter-loader": "3.0.1",
+"karma-source-map-support": "1.3.0",
+"less": "3.8.1",
+"less-loader": "4.1.0",
+"license-webpack-plugin": "1.5.0",
+"loader-utils": "1.1.0",
+"mini-css-extract-plugin": "0.4.3",
+"minimatch": "3.0.4",
+"node-sass": "4.9.3",
+"opn": "5.3.0",
+"parse5": "4.0.0",
+"portfinder": "1.0.17",
+"postcss": "6.0.23",
+"postcss-import": "11.1.0",
+"postcss-loader": "2.1.6",
+"postcss-url": "7.3.2",
+"raw-loader": "0.5.1",
+"rxjs": "6.2.2",
+"sass-loader": "7.1.0",
+"semver": "5.5.1",
+"source-map-loader": "0.2.4",
+"source-map-support": "0.5.9",
+"stats-webpack-plugin": "0.6.2",
+"style-loader": "0.21.0",
+"stylus": "0.54.5",
+"stylus-loader": "3.0.2",
+"tree-kill": "1.2.0",
+"uglifyjs-webpack-plugin": "1.3.0",
+"url-loader": "1.1.1",
+"webpack": "4.19.1",
+"webpack-dev-middleware": "3.3.0",
+"webpack-dev-server": "3.1.8",
+"webpack-merge": "4.1.4",
+"webpack-sources": "1.3.0",
+"webpack-subresource-integrity": "1.1.0-rc.6"
+ },
+ "dependencies": {
+"@angular-devkit/core": {
+ "version": "0.8.3",
+ "resolved":
"https://registry.npmjs.org/@angular-devkit/core/-/core-0.8.3.tgz;,
+ "integrity":
"sha512-2KHt5osMs3zACYXev20ZU5SXdWoinoKwZkj2caj2LCj9W7QNHmsz34QvaygNq7YdJzF3jkXkdy0GSUgUgDke0w==",
+ "dev": true,
+ "requires": {
+"ajv": "6.4.0",
+"chokidar": "2.0.4",
+"rxjs": "6.2.2",
+"source-map": "0.5.7"
+ }
+}
+ }
+},
+"@angular-devkit/build-optimizer": {
+ "version": "0.8.3",
+
[02/52] [abbrv] metron git commit: METRON-1803: Integrate Cypress with Travis (tiborm via mmiklavc) closes apache/metron#1226
http://git-wip-us.apache.org/repos/asf/metron/blob/9b6260fd/metron-interface/metron-alerts/cypress/fixtures/search.json
--
diff --git a/metron-interface/metron-alerts/cypress/fixtures/search.json
b/metron-interface/metron-alerts/cypress/fixtures/search.json
new file mode 100644
index 000..e2e03e4
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress/fixtures/search.json
@@ -0,0 +1,5647 @@
+{
+ "total":104593,
+ "results":[
+ {
+"id":"ad5cc7ea-5954-479f-8589-51f94b1c2f02",
+"source":{
+ "average":10.0,
+ "max":10.0,
+ "metron_alert":[
+ {
+ "msg":"'snort test alert'",
+ "sig_rev":"0",
+ "ip_dst_port":"49195",
+ "threatinteljoinbolt:joiner:ts":"1537279364136",
+ "ethsrc":"00:00:00:00:00:00",
+ "threat:triage:rules:0:comment":null,
+ "tcpseq":"0xC88832BC",
+ "enrichments:geo:ip_src_addr:longitude":"2.33870002",
+ "dgmlen":"44",
+ "enrichmentsplitterbolt:splitter:begin:ts":"1537279364122",
+ "enrichmentjoinbolt:joiner:ts":"1537279364128",
+ "adapter:geoadapter:begin:ts":"1537279364125",
+ "tcpwindow":"0xFAF0",
+ "threat:triage:rules:0:name":null,
+ "tcpack":"0x522C98B4",
+ "protocol":"TCP",
+ "source:type":"snort",
+ "adapter:threatinteladapter:end:ts":"1537279364133",
+ "ip_dst_addr":"192.168.138.158",
+ "original_string":"09/18/18-14:02:39.00
,1,999158,0,\"'snort test
alert'\",TCP,188.165.164.184,80,192.168.138.158,49195,00:00:00:00:00:00,00:00:00:00:00:00,0x3C,***A**S*,0xC88832BC,0x522C98B4,,0xFAF0,128,0,1899,44,45056",
+ "adapter:hostfromjsonlistadapter:end:ts":"1537279364125",
+ "tos":"0",
+ "adapter:geoadapter:end:ts":"1537279364125",
+ "id":"1899",
+ "enrichments:geo:ip_src_addr:latitude":"48.8582",
+ "ip_src_addr":"188.165.164.184",
+ "threatintelsplitterbolt:splitter:end:ts":"1537279364130",
+ "threat:triage:rules:0:score":10,
+ "timestamp":1537279359000,
+ "ethdst":"00:00:00:00:00:00",
+
"enrichments:geo:ip_src_addr:location_point":"48.8582,2.33870002",
+ "threat:triage:rules:0:reason":null,
+ "enrichmentsplitterbolt:splitter:end:ts":"1537279364122",
+ "threat:triage:score":10.0,
+ "is_alert":"true",
+ "adapter:hostfromjsonlistadapter:begin:ts":"1537279364125",
+ "enrichments:geo:ip_src_addr:country":"FR",
+ "ttl":"128",
+ "metaalerts":[
+"ad5cc7ea-5954-479f-8589-51f94b1c2f02"
+ ],
+ "ethlen":"0x3C",
+ "iplen":"45056",
+ "ip_src_port":"80",
+ "threatintelsplitterbolt:splitter:begin:ts":"1537279364130",
+ "adapter:threatinteladapter:begin:ts":"1537279364133",
+ "tcpflags":"***A**S*",
+ "guid":"c6843745-203c-49e1-80ad-f060eb88c9b1",
+ "sig_id":"999158",
+ "sig_generator":"1"
+ },
+ {
+ "msg":"'snort test alert'",
+ "sig_rev":"0",
+ "ip_dst_port":"49195",
+ "threatinteljoinbolt:joiner:ts":"1537280091506",
+ "ethsrc":"00:00:00:00:00:00",
+ "threat:triage:rules:0:comment":null,
+ "tcpseq":"0xC88832BD",
+ "enrichments:geo:ip_src_addr:longitude":"2.33870002",
+ "dgmlen":"40",
+ "enrichmentsplitterbolt:splitter:begin:ts":"1537280091491",
+ "enrichmentjoinbolt:joiner:ts":"1537280091498",
+ "adapter:geoadapter:begin:ts":"1537280091493",
+ "tcpwindow":"0xFAF0",
+ "threat:triage:rules:0:name":null,
+ "tcpack":"0x522C999D",
+ "protocol":"TCP",
+ "source:type":"snort",
+ "adapter:threatinteladapter:end:ts":"1537280091503",
+ "ip_dst_addr":"192.168.138.158",
+ "original_string":"09/18/18-14:14:47.00
,1,999158,0,\"'snort test
alert'\",TCP,188.165.164.184,80,192.168.138.158,49195,00:00:00:00:00:00,00:00:00:00:00:00,0x3C,***A,0xC88832BD,0x522C999D,,0xFAF0,128,0,1900,40,40960",
+ "adapter:hostfromjsonlistadapter:end:ts":"1537280091493",
+ "tos":"0",
+ "adapter:geoadapter:end:ts":"1537280091493",
+ "id":"1900",
+
[50/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc)
METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fcd644ca Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fcd644ca Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fcd644ca Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: fcd644ca77394d48d460c460b672a23d6594f49b Parents: 0c4c622 8bf3b6e Author: mmiklavc Authored: Thu Nov 15 17:03:18 2018 -0700 Committer: Michael Miklavcic Committed: Thu Nov 15 17:03:55 2018 -0700 -- Upgrading.md| 7 + dependencies_with_url.csv | 2 + metron-deployment/Kerberos-manual-setup.md | 154 +--- .../METRON/CURRENT/configuration/metron-env.xml | 9 - .../CURRENT/package/scripts/metron_service.py | 2 - .../package/scripts/params/params_linux.py | 3 +- .../METRON/CURRENT/themes/metron_theme.json | 10 - .../rest/service/impl/MetaAlertServiceImpl.java | 2 +- metron-platform/elasticsearch-shaded/pom.xml| 47 +--- .../META-INF/log4j-provider.properties | 18 -- metron-platform/metron-common/README.md | 48 ++-- .../src/main/config/zookeeper/global.json | 1 - .../common/configuration/ConfigOption.java | 7 + metron-platform/metron-elasticsearch/README.md | 45 +++- metron-platform/metron-elasticsearch/pom.xml| 32 ++- .../client/ElasticsearchClient.java | 245 +++ .../client/ElasticsearchClientFactory.java | 189 ++ .../config/ElasticsearchClientConfig.java | 187 ++ .../config/ElasticsearchClientOptions.java | 60 + .../dao/ElasticsearchColumnMetadataDao.java | 101 +++- .../elasticsearch/dao/ElasticsearchDao.java | 21 +- .../dao/ElasticsearchMetaAlertDao.java | 2 +- .../dao/ElasticsearchMetaAlertSearchDao.java| 6 +- .../dao/ElasticsearchMetaAlertUpdateDao.java| 4 +- .../dao/ElasticsearchRequestSubmitter.java | 13 +- .../dao/ElasticsearchRetrieveLatestDao.java | 28 ++- .../dao/ElasticsearchSearchDao.java | 19 +- .../dao/ElasticsearchUpdateDao.java | 19 +- .../elasticsearch/utils/ElasticsearchUtils.java | 182 ++ .../elasticsearch/utils/FieldMapping.java | 32 +++ .../elasticsearch/utils/FieldProperties.java| 36 +++ .../writer/ElasticsearchWriter.java | 26 +- .../dao/ElasticsearchColumnMetadataDaoTest.java | 59 ++--- .../elasticsearch/dao/ElasticsearchDaoTest.java | 8 +- .../dao/ElasticsearchRequestSubmitterTest.java | 23 +- .../dao/ElasticsearchUpdateDaoTest.java | 12 +- .../ElasticsearchMetaAlertIntegrationTest.java | 9 +- .../ElasticsearchSearchIntegrationTest.java | 144 ++- .../ElasticsearchUpdateIntegrationTest.java | 2 +- .../components/ElasticSearchComponent.java | 6 +- .../dao/metaalert/MetaAlertSearchDao.java | 4 +- .../dao/metaalert/MetaAlertIntegrationTest.java | 2 +- .../src/main/config/zookeeper/global.json | 2 +- 43 files changed, 1134 insertions(+), 694 deletions(-) --
[44/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-alerts/cypress/integration/pcap/pcap.spec.js
--
diff --git
a/metron-interface/metron-alerts/cypress/integration/pcap/pcap.spec.js
b/metron-interface/metron-alerts/cypress/integration/pcap/pcap.spec.js
new file mode 100644
index 000..58f7d26
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress/integration/pcap/pcap.spec.js
@@ -0,0 +1,228 @@
+///
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+context('PCAP Tab', () => {
+
+ beforeEach(() => {
+cy.server();
+cy.route({
+ method: 'GET',
+ url: '/api/v1/user',
+ response: 'user'
+});
+
+cy.route('GET', 'config', 'fixture:config.json');
+cy.route('POST', 'search', 'fixture:search.json');
+
+cy.route({
+ method: 'GET',
+ url: '/api/v1/pcap?state=*',
+ response: []
+}).as('runningJobs');
+
+cy.visit('http://localhost:4200/login');
+cy.get('[name="user"]').type('user');
+cy.get('[name="password"]').type('password');
+cy.contains('LOG IN').click();
+ });
+
+ afterEach(() => {
+cy.get('.logout-link').click();
+ });
+
+ it('checking running jobs on navigating to PCAP tab', () => {
+cy.contains('PCAP').click();
+cy.wait('@runningJobs').its('url').should('include', '?state=RUNNING');
+ });
+
+ it('submitting PCAP job request', () => {
+cy.contains('PCAP').click();
+cy.route('POST', '/api/v1/pcap/fixed', 'fixture:pcap.status-00.json')
+ .as('postingPcapJob');
+
+cy.get('[data-qe-id="ip-src-addr"]').type('222.123.111.000');
+cy.get('[data-qe-id="ip-dst-addr"]').type('111.123.222.000');
+cy.get('[data-qe-id="ip-src-port"]').type('');
+cy.get('[data-qe-id="ip-dst-port"]').type('');
+cy.get('[data-qe-id="protocol"]').type('24');
+cy.get('[data-qe-id="include-reverse"]').check();
+cy.get('[data-qe-id="packet-filter"]').type('filter');
+
+cy.get('[data-qe-id="submit-button"]').click();
+
+cy.wait('@postingPcapJob').then((xhr) => {
+ expect(xhr.request.body.ipSrcAddr).to.equal('222.123.111.000');
+ expect(xhr.request.body.ipDstAddr).to.equal('111.123.222.000');
+ expect(xhr.request.body.ipSrcPort).to.equal('');
+ expect(xhr.request.body.ipDstPort).to.equal('');
+ expect(xhr.request.body.protocol).to.equal('24');
+ expect(xhr.request.body.includeReverse).to.equal(true);
+ expect(xhr.request.body.packetFilter).to.equal('filter');
+});
+ });
+
+ it('requesting job status', () => {
+cy.route('POST', '/api/v1/pcap/fixed', 'fixture:pcap.status-00.json');
+cy.route('GET', '/api/v1/pcap/*',
'fixture:pcap.status-01.json').as('jobStatusCheck');
+
+cy.contains('PCAP').click();
+cy.get('[data-qe-id="submit-button"]').click();
+
+cy.wait('@jobStatusCheck').its('url').should('include',
'/api/v1/pcap/job_1537878471649_0001');
+ });
+
+ it('process status in percentage', () => {
+cy.route('POST', '/api/v1/pcap/fixed', 'fixture:pcap.status-00.json');
+cy.route('GET', '/api/v1/pcap/*',
'fixture:pcap.status-01.json').as('jobStatusCheck');
+
+cy.contains('PCAP').click();
+cy.get('[data-qe-id="submit-button"]').click();
+
+cy.wait('@jobStatusCheck');
+
+cy.contains('75%').should('be.visible');
+ });
+
+ it('getting pcap json', () => {
+cy.route('POST', '/api/v1/pcap/fixed', 'fixture:pcap.status-00.json');
+cy.route('GET', '/api/v1/pcap/*',
'fixture:pcap.status-02.json').as('statusCheck');
+cy.route('GET', '/api/v1/pcap/*/pdml*',
'fixture:pcap.page-01.json').as('gettingPdml');
+
+cy.contains('PCAP').click();
+cy.get('[data-qe-id="submit-button"]').click();
+
+cy.wait('@statusCheck');
+
+cy.wait('@gettingPdml').its('url').should('include',
'/api/v1/pcap/job_1537878471649_0001/pdml?page=1');
+ });
+
+
+ it('rendering pcap table', () => {
+cy.route('POST', '/api/v1/pcap/fixed', 'fixture:pcap.status-00.json');
+cy.route('GET', '/api/v1/pcap/*',
'fixture:pcap.status-02.json').as('statusCheck');
+cy.route('GET', '/api/v1/pcap/*/pdml*',
[51/52] [abbrv] metron git commit: METRON-1875 Expose configurable global settings in the Alerts UI (merrimanr) closes apache/metron#1266
METRON-1875 Expose configurable global settings in the Alerts UI (merrimanr)
closes apache/metron#1266
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3c4c069b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3c4c069b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3c4c069b
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 3c4c069b6f29824122dd93ac6f234ef1d0363e9d
Parents: fcd644c
Author: merrimanr
Authored: Fri Nov 16 14:38:44 2018 -0600
Committer: rmerriman
Committed: Fri Nov 16 14:38:44 2018 -0600
--
.../docker/rpm-docker/SPECS/metron.spec | 1 +
.../table-view/table-view.component.spec.ts | 11 +-
.../tree-view/tree-view.component.spec.ts | 11 +-
.../metron-alerts/src/app/app.module.ts | 11 +++---
.../pcap-panel/pcap-panel.component.spec.ts | 9 +
.../src/app/pcap/service/pcap.service.spec.ts | 13 ++-
.../src/app/pcap/service/pcap.service.ts| 17
.../src/app/service/alerts.service.ts | 5 ++-
.../src/app/service/app-config.service.ts | 41
.../src/app/service/authentication.service.ts | 10 +++--
.../src/app/service/column-names.service.ts | 4 --
.../src/app/service/global-config.service.ts| 5 ++-
.../src/app/service/meta-alert.service.ts | 11 +++---
.../src/app/service/search.service.ts | 11 +++---
.../src/app/service/update.service.ts | 9 +++--
.../metron-alerts/src/assets/app-config.json| 3 ++
16 files changed, 129 insertions(+), 43 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/3c4c069b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 14c08a1..2839ffa 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -540,6 +540,7 @@ This package installs the Metron Alerts UI %{metron_home}
%attr(0644,root,root)
%{metron_home}/web/alerts-ui/assets/fonts/Roboto/LICENSE.txt
%attr(0644,root,root) %{metron_home}/web/alerts-ui/assets/fonts/Roboto/*.ttf
%attr(0644,root,root) %{metron_home}/web/alerts-ui/assets/images/*
+%attr(0644,root,root) %{metron_home}/web/alerts-ui/assets/app-config.json
# ~~
http://git-wip-us.apache.org/repos/asf/metron/blob/3c4c069b/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.spec.ts
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.spec.ts
index 2d7af57..8f2b4c4 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.spec.ts
@@ -31,12 +31,20 @@ import { UpdateService } from
'../../../service/update.service';
import { GlobalConfigService } from '../../../service/global-config.service';
import { MetaAlertService } from '../../../service/meta-alert.service';
import { DialogService } from 'app/service/dialog.service';
+import { AppConfigService } from '../../../service/app-config.service';
@Component({selector: 'metron-table-pagination', template: ''})
class MetronTablePaginationComponent {
@Input() pagination = 0;
}
+class FakeAppConfigService {
+
+ getApiRoot() {
+return '/api/v1'
+ }
+}
+
describe('TableViewComponent', () => {
let component: TableViewComponent;
let fixture: ComponentFixture;
@@ -50,7 +58,8 @@ describe('TableViewComponent', () => {
UpdateService,
GlobalConfigService,
MetaAlertService,
-DialogService
+DialogService,
+{ provide: AppConfigService, useClass: FakeAppConfigService }
],
declarations: [
MetronTableDirective,
http://git-wip-us.apache.org/repos/asf/metron/blob/3c4c069b/metron-interface/metron-alerts/src/app/alerts/alerts-list/tree-view/tree-view.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/tree-view/tree-view.component.spec.ts
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/tree-view/tree-view.component.spec.ts
index fda2b78..3fc145f 100644
---
[33/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-enrichment/README.md
--
diff --git a/metron-platform/metron-enrichment/README.md
b/metron-platform/metron-enrichment/README.md
index 8a53e71..c72970f 100644
--- a/metron-platform/metron-enrichment/README.md
+++ b/metron-platform/metron-enrichment/README.md
@@ -31,36 +31,22 @@ data format (e.g. a JSON Map structure with
`original_message` and
## Enrichment Architecture
-
+
### Unified Enrichment Topology
-There is an experimental unified enrichment topology which is shipped.
-Currently the architecture, as described above, has a split/join in
-order to perform enrichments in parallel. This poses some issues in
-terms of ease of tuning and reasoning about performance.
-
-In order to deal with these issues, there is an alternative enrichment
topology which
-uses data parallelism as opposed to the split/join task parallelism.
-This architecture uses a worker pool to fully enrich any message within
-a worker. This results in
+The unified enrichment topology uses data parallelism as opposed to the
deprecated
+split/join topology's task parallelism. This architecture uses a worker pool
to fully
+enrich any message within a worker. This results in
* Fewer bolts in the topology
* Each bolt fully operates on a message.
* Fewer network hops
-
-
-This architecture is fully backwards compatible; the only difference is
-how the enrichment will operate on each message (in one bolt where the
-split/join is done in a threadpool as opposed
+This architecture is fully backwards compatible with the old split-join
+topology; the only difference is how the enrichment will operate on each
+message (in one bolt where the split/join is done in a threadpool as opposed
to split across multiple bolts).
- Using It
-
-In order to use this, you will need to
-* Edit `$METRON_HOME/bin/start_enrichment_topology.sh` and adjust it to use
`remote-unified.yaml` instead of `remote.yaml`
-* Restart the enrichment topology.
-
Configuring It
There are two parameters which you might want to tune in this topology.
@@ -76,6 +62,19 @@ intel bolt, the configurations will be taken from the
respective join bolt
parallelism. When proper ambari support for this is added, we will add
its own property.
+### Split-Join Enrichment Topology
+
+The now-deprecated split/join topology is also available and performs
enrichments in parallel.
+This poses some issues in terms of ease of tuning and reasoning about
performance.
+
+
+
+ Using It
+
+In order to use the older, deprecated topology, you will need to
+* Edit `$METRON_HOME/bin/start_enrichment_topology.sh` and adjust it to use
`remote-splitjoin.yaml` instead of `remote-unified.yaml`
+* Restart the enrichment topology.
+
## Enrichment Configuration
The configuration for the `enrichment` topology, the topology primarily
@@ -85,7 +84,6 @@ defined by JSON documents stored in zookeeper.
There are two types of configurations at the moment, `global` and
`sensor` specific.
-
## Global Configuration
There are a few enrichments which have independent configurations, such
@@ -134,7 +132,6 @@ The configuration is a complex JSON object with the
following top level fields:
### The `enrichment` Configuration
-
| Field| Description
| Example |
|--|---|--|
| `fieldToTypeMap` | In the case of a simple HBase enrichment (i.e. a
key/value lookup), the mapping between fields and the enrichment types
associated with those fields must be known. This enrichment type is used as
part of the HBase key. Note: applies to hbaseEnrichment only. |
`"fieldToTypeMap" : { "ip_src_addr" : [ "asset_enrichment" ] }` |
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
--
diff --git
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/GenericEnrichmentBolt.java
[31/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
--
diff --git
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
new file mode 100644
index 000..e24a39d
--- /dev/null
+++
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/MultiLineGrokParserTest.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class MultiLineGrokParserTest {
+
+ /**
+ * Test that if a byte[] with multiple lines of log is passed in
+ * it will be parsed into the correct number of messages.
+ * @throws IOException if we can't read from disk
+ * @throws ParseException if we can't parse
+ */
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testLegacyInterfaceReturnsMultiline() throws IOException,
ParseException {
+
+Map parserConfig = new HashMap<>();
+parserConfig.put("grokPath", getGrokPath());
+parserConfig.put("patternLabel", getGrokPatternLabel());
+parserConfig.put("timestampField", getTimestampField());
+parserConfig.put("dateFormat", getDateFormat());
+parserConfig.put("timeFields", getTimeFields());
+parserConfig.put("multiLine", getMultiLine());
+GrokParser grokParser = new GrokParser();
+grokParser.configure(parserConfig);
+grokParser.init();
+
+JSONParser jsonParser = new JSONParser();
+Map testData = getTestData();
+for (Map.Entry e : testData.entrySet()) {
+ byte[] rawMessage = e.getKey().getBytes();
+ Optional> resultOptional =
grokParser.parseOptionalResult(rawMessage);
+ Assert.assertNotNull(resultOptional);
+ Assert.assertTrue(resultOptional.isPresent());
+ List parsedList = resultOptional.get().getMessages();
+ Assert.assertEquals(10, parsedList.size());
+}
+ }
+
+ /**
+ * Test that if a byte[] with multiple lines of log is passed in
+ * it will be parsed into the correct number of messages using the
+ * parseOptionalResult call.
+ * @throws IOException if we can't read from disk
+ * @throws ParseException if we can't parse
+ */
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testOptionalResultReturnsMultiline() throws IOException,
ParseException {
+
+Map parserConfig = new HashMap<>();
+parserConfig.put("grokPath", getGrokPath());
+parserConfig.put("patternLabel", getGrokPatternLabel());
+parserConfig.put("timestampField", getTimestampField());
+parserConfig.put("dateFormat", getDateFormat());
+parserConfig.put("timeFields", getTimeFields());
+parserConfig.put("multiLine", getMultiLine());
+
+GrokParser grokParser = new GrokParser();
+grokParser.configure(parserConfig);
+grokParser.init();
+
+JSONParser jsonParser = new JSONParser();
+Map testData = getTestData();
+for (Map.Entry e : testData.entrySet()) {
+ byte[] rawMessage = e.getKey().getBytes();
+ Optional> resultOptional =
grokParser.parseOptionalResult(rawMessage);
+ Assert.assertTrue(resultOptional.isPresent());
+ Optional throwableOptional =
resultOptional.get().getMasterThrowable();
+ List resultList = resultOptional.get().getMessages();
+ Map errorMap =
resultOptional.get().getMessageThrowables();
+ Assert.assertFalse(throwableOptional.isPresent());
+ Assert.assertEquals(0, errorMap.size());
+ Assert.assertEquals(10, resultList.size());
+}
+ }
+
+
[04/52] [abbrv] metron git commit: METRON-1803: Integrate Cypress with Travis (tiborm via mmiklavc) closes apache/metron#1226
http://git-wip-us.apache.org/repos/asf/metron/blob/9b6260fd/metron-interface/metron-alerts/cypress/fixtures/pcap.page-01.json
--
diff --git a/metron-interface/metron-alerts/cypress/fixtures/pcap.page-01.json
b/metron-interface/metron-alerts/cypress/fixtures/pcap.page-01.json
new file mode 100644
index 000..61082ed
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress/fixtures/pcap.page-01.json
@@ -0,0 +1,12383 @@
+{
+ "version":"0",
+ "creator":"wireshark/1.8.10",
+ "time":"Mon Sep 24 14:16:26 2018",
+ "captureFile":"",
+ "packets":[
+ {
+"protos":[
+ {
+ "name":"geninfo",
+ "pos":"0",
+ "showname":"General information",
+ "size":"722",
+ "hide":null,
+ "fields":[
+ {
+"name":"num",
+"pos":"0",
+"showname":"Number",
+"size":"722",
+"value":"1",
+"show":"1",
+"unmaskedvalue":null,
+"hide":null,
+"fields":null,
+"protos":null
+ },
+ {
+"name":"len",
+"pos":"0",
+"showname":"Frame Length",
+"size":"722",
+"value":"2d2",
+"show":"722",
+"unmaskedvalue":null,
+"hide":null,
+"fields":null,
+"protos":null
+ },
+ {
+"name":"caplen",
+"pos":"0",
+"showname":"Captured Length",
+"size":"722",
+"value":"2d2",
+"show":"722",
+"unmaskedvalue":null,
+"hide":null,
+"fields":null,
+"protos":null
+ },
+ {
+"name":"timestamp",
+"pos":"0",
+"showname":"Captured Time",
+"size":"722",
+"value":"1458240269.373968000",
+"show":"Mar 17, 2016 18:44:29.373968000 UTC",
+"unmaskedvalue":null,
+"hide":null,
+"fields":null,
+"protos":null
+ }
+ ]
+ },
+ {
+ "name":"frame",
+ "pos":"0",
+ "showname":"Frame 1: 722 bytes on wire (5776 bits), 722 bytes
captured (5776 bits) on interface 0",
+ "size":"722",
+ "hide":null,
+ "fields":[
+ {
+"name":"frame.interface_id",
+"pos":"0",
+"showname":"Interface id: 0",
+"size":"0",
+"value":null,
+"show":"0",
+"unmaskedvalue":null,
+"hide":null,
+"fields":null,
+"protos":null
+ },
+ {
+"name":"frame.dlt",
+"pos":"0",
+"showname":"WTAP_ENCAP: 1",
+"size":"0",
+"value":null,
+"show":"1",
+"unmaskedvalue":null,
+"hide":null,
+"fields":null,
+"protos":null
+ },
+ {
+"name":"frame.time",
+"pos":"0",
+"showname":"Arrival Time: Mar 17, 2016 18:44:29.373968000
UTC",
+"size":"0",
+"value":null,
+"show":"Mar 17, 2016 18:44:29.373968000",
+"unmaskedvalue":null,
+"hide":null,
+"fields":null,
+"protos":null
+ },
+ {
+"name":"frame.offset_shift",
+"pos":"0",
+"showname":"Time shift for this packet: 0.0
seconds",
+"size":"0",
+"value":null,
+"show":"0.0",
+"unmaskedvalue":null,
+"hide":null,
+"fields":null,
+"protos":null
+ },
+ {
+"name":"frame.time_epoch",
+"pos":"0",
+"showname":"Epoch Time: 1458240269.373968000 seconds",
+"size":"0",
+"value":null,
+"show":"1458240269.373968000",
+"unmaskedvalue":null,
+
[08/52] [abbrv] metron git commit: METRON-1857 Fix Metaalert Nested Alert Field Name in Index Template (nickwallen) closes apache/metron#1253
METRON-1857 Fix Metaalert Nested Alert Field Name in Index Template
(nickwallen) closes apache/metron#1253
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b9461e76
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b9461e76
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b9461e76
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: b9461e76506988ad51d6fc0f3a442909945048f2
Parents: bf6b07f
Author: nickwallen
Authored: Tue Nov 6 10:04:48 2018 -0500
Committer: nickallen
Committed: Tue Nov 6 10:04:48 2018 -0500
--
.../METRON/CURRENT/package/files/metaalert_index.template| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/b9461e76/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
index 05d5e32..040c411 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template
@@ -5,8 +5,8 @@
"dynamic_templates": [
{
"alert_template": {
- "path_match": "alert.*",
- "match_mapping_type": "string",
+ "path_match": "metron_alert.*",
+ "match_mapping_type": "*",
"mapping": {
"type": "keyword"
}
[13/52] [abbrv] metron git commit: METRON-1850 Stellar REST function (merrimanr) closes apache/metron#1250
METRON-1850 Stellar REST function (merrimanr) closes apache/metron#1250
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3e73391e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3e73391e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3e73391e
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 3e73391ed518e6d4e2f87a37745a34581782236b
Parents: b5712af
Author: merrimanr
Authored: Thu Nov 8 16:44:32 2018 -0600
Committer: rmerriman
Committed: Thu Nov 8 16:44:32 2018 -0600
--
metron-analytics/metron-maas-common/pom.xml | 2 +-
.../common/bolt/ConfiguredEnrichmentBolt.java | 14 +
.../common/bolt/ConfiguredParserBolt.java | 13 +
metron-platform/metron-data-management/pom.xml | 5 +-
.../enrichment/bolt/GenericEnrichmentBolt.java | 1 +
.../enrichment/bolt/ThreatIntelJoinBolt.java| 6 -
metron-stellar/stellar-common/README.md | 98 ++-
metron-stellar/stellar-common/pom.xml | 40 ++
.../stellar/common/shell/cli/StellarShell.java | 2 +
.../org/apache/metron/stellar/dsl/Context.java | 5 +
.../stellar/dsl/functions/RestConfig.java | 166 +
.../stellar/dsl/functions/RestFunctions.java| 388
.../dsl/functions/RestFunctionsTest.java| 604 +++
pom.xml | 1 +
14 files changed, 1334 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/3e73391e/metron-analytics/metron-maas-common/pom.xml
--
diff --git a/metron-analytics/metron-maas-common/pom.xml
b/metron-analytics/metron-maas-common/pom.xml
index b5cf1c9..062e3db 100644
--- a/metron-analytics/metron-maas-common/pom.xml
+++ b/metron-analytics/metron-maas-common/pom.xml
@@ -62,7 +62,7 @@
org.apache.httpcomponents
httpclient
- 4.3.2
+ ${global_httpclient_version}
org.codehaus.jackson
http://git-wip-us.apache.org/repos/asf/metron/blob/3e73391e/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java
--
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java
index c28ca7b..2e03a36 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java
@@ -17,8 +17,10 @@
*/
package org.apache.metron.common.bolt;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.stellar.dsl.StellarFunctions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,4 +33,16 @@ public abstract class ConfiguredEnrichmentBolt extends
ConfiguredBolthttps://storm.apache.org/releases/1.0.6/javadocs/org/apache/storm/task/IBolt.html#cleanup--
for more detail.
+super.cleanup();
+try {
+ StellarFunctions.close();
+} catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+}
+ }
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3e73391e/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java
--
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java
index 14ce50b..17b614b 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java
@@ -17,9 +17,11 @@
*/
package org.apache.metron.common.bolt;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.stellar.dsl.StellarFunctions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,4 +38,15 @@ public abstract class ConfiguredParserBolt extends
ConfiguredBolthttps://storm.apache.org/releases/1.0.6/javadocs/org/apache/storm/task/IBolt.html#cleanup--
for more detail.
+super.cleanup();
+try {
+ StellarFunctions.close();
+} catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+}
[05/52] [abbrv] metron git commit: METRON-1803: Integrate Cypress with Travis (tiborm via mmiklavc) closes apache/metron#1226
METRON-1803: Integrate Cypress with Travis (tiborm via mmiklavc) closes
apache/metron#1226
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9b6260fd
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9b6260fd
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9b6260fd
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 9b6260fd4b9ecf355864b77c8889d27539623381
Parents: fefbb37
Author: tiborm
Authored: Fri Nov 2 10:21:41 2018 -0600
Committer: Michael Miklavcic
Committed: Fri Nov 2 10:21:41 2018 -0600
--
metron-interface/metron-alerts/cypress.json | 7 +
.../metron-alerts/cypress/fixtures/config.json |23 +
.../cypress/fixtures/pcap.page-01.json | 12383 +
.../cypress/fixtures/pcap.status-00.json| 7 +
.../cypress/fixtures/pcap.status-01.json| 7 +
.../cypress/fixtures/pcap.status-02.json| 7 +
.../metron-alerts/cypress/fixtures/search.json | 5647
.../cypress/integration/pcap/pcap.spec.js | 228 +
.../metron-alerts/package-lock.json | 1134 ++
metron-interface/metron-alerts/package.json |11 +-
10 files changed, 19451 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/9b6260fd/metron-interface/metron-alerts/cypress.json
--
diff --git a/metron-interface/metron-alerts/cypress.json
b/metron-interface/metron-alerts/cypress.json
new file mode 100644
index 000..7c0410d
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress.json
@@ -0,0 +1,7 @@
+{
+ "viewportWidth": 1435,
+ "viewportHeight": 850,
+ "video": false,
+ "supportFile": false,
+ "pluginsFile": false
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/9b6260fd/metron-interface/metron-alerts/cypress/fixtures/config.json
--
diff --git a/metron-interface/metron-alerts/cypress/fixtures/config.json
b/metron-interface/metron-alerts/cypress/fixtures/config.json
new file mode 100644
index 000..190e514
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress/fixtures/config.json
@@ -0,0 +1,23 @@
+{
+ "es.clustername":"metron",
+ "es.ip":"node1:9300",
+ "es.date.format":".MM.dd.HH",
+ "parser.error.topic":"indexing",
+ "update.hbase.table":"metron_update",
+ "update.hbase.cf":"t",
+ "es.client.settings":{
+ "client.transport.ping_timeout":"500s"
+ },
+ "profiler.client.period.duration":"15",
+ "profiler.client.period.duration.units":"MINUTES",
+ "user.settings.hbase.table":"user_settings",
+ "user.settings.hbase.cf":"cf",
+ "bootstrap.servers":"node1:6667",
+ "source.type.field":"source:type",
+ "threat.triage.score.field":"threat:triage:score",
+ "enrichment.writer.batchSize":"15",
+ "enrichment.writer.batchTimeout":"0",
+ "profiler.writer.batchSize":"15",
+ "profiler.writer.batchTimeout":"0",
+ "geo.hdfs.file":"/apps/metron/geo/default/GeoLite2-City.mmdb.gz"
+}
\ No newline at end of file
[24/52] [abbrv] metron git commit: METRON-1749 Update Angular to latest release in Management UI (sardell via nickwallen) closes apache/metron#1217
http://git-wip-us.apache.org/repos/asf/metron/blob/0c4c622b/metron-interface/metron-config/package.json
--
diff --git a/metron-interface/metron-config/package.json
b/metron-interface/metron-config/package.json
index 6e196d5..ffc92ff 100644
--- a/metron-interface/metron-config/package.json
+++ b/metron-interface/metron-config/package.json
@@ -7,10 +7,11 @@
},
"angular-cli": {},
"scripts": {
-"build": "./node_modules/angular-cli/bin/ng build -prod",
+"build": "ng build --prod",
"start": "ng serve",
"lint": "tslint \"src/**/*.ts\"",
-"test": "./node_modules/angular-cli/bin/ng test --watch=false",
+"test": "ng test --browsers=Chrome",
+"testCI": "ng test --watch=false --browsers=ChromeHeadless",
"pree2e": "webdriver-manager update",
"e2e": "./node_modules/.bin/protractor",
"e2e-all": "./node_modules/.bin/protractor --suite=all",
@@ -20,47 +21,54 @@
},
"private": true,
"dependencies": {
+"@angular-devkit/schematics": "^0.7.5",
+"@angular/cli": "^6.2.3",
+"@angular/common": "^6.1.8",
+"@angular/compiler": "^6.1.8",
+"@angular/core": "^6.1.8",
+"@angular/forms": "^6.1.8",
+"@angular/http": "^6.1.8",
+"@angular/platform-browser": "^6.1.8",
+"@angular/platform-browser-dynamic": "^6.1.8",
+"@angular/platform-server": "^6.1.8",
+"@angular/router": "^6.1.8",
"@types/ace": "0.0.32",
-"@types/bootstrap": "^3.3.32",
-"@types/jasmine": "2.2.30",
-"@types/jquery": "^2.0.32",
+"@types/bootstrap": "^4.1.2",
+"@types/jasmine": "~2.8.6",
+"@types/jasminewd2": "~2.0.3",
+"@types/jquery": "^3.3.6",
+"@types/node": "^10.9.4",
"@types/tether": "^1.1.27",
-"@angular/common": "2.0.0",
-"@angular/compiler": "2.0.0",
-"@angular/core": "2.0.0",
-"@angular/forms": "2.0.0",
-"@angular/http": "2.0.0",
-"@angular/platform-browser": "2.0.0",
-"@angular/platform-browser-dynamic": "2.0.0",
-"@angular/router": "3.0.0",
"ace-builds": "^1.2.5",
"bootstrap": "4.0.0-alpha.5",
-"core-js": "^2.4.1",
+"core-js": "^2.5.7",
"font-awesome": "^4.6.3",
"jquery": "^3.3.1",
-"rxjs": "5.0.0-beta.12",
+"karma-phantomjs-launcher": "^1.0.4",
+"puppeteer": "^1.8.0",
+"rxjs": "6.2.2",
"tether": "^1.3.4",
"ts-helpers": "^1.1.1",
-"zone.js": "^0.6.23"
+"zone.js": "^0.8.26"
},
"devDependencies": {
-"angular-cli": "1.0.0-beta.15",
+"@angular-devkit/build-angular": "^0.8.3",
+"@angular/compiler-cli": "^6.1.8",
+"@types/request": "2.0.3",
"buffer-shims": "^1.0.0",
-"codelyzer": "~0.0.26",
+"codelyzer": "~4.2.1",
"copy": "^0.3.0",
-"jasmine-core": "2.4.1",
-"jasmine-spec-reporter": "2.5.0",
-"karma": "1.2.0",
-"karma-chrome-launcher": "^2.0.0",
-"karma-cli": "^1.0.1",
-"karma-jasmine": "^1.0.2",
-"karma-phantomjs-launcher": "^1.0.4",
-"karma-remap-istanbul": "0.6.0",
+"jasmine-core": "~2.99.1",
+"jasmine-spec-reporter": "~4.2.1",
+"karma": "^3.0.0",
+"karma-chrome-launcher": "~2.2.0",
+"karma-coverage-istanbul-reporter": "~2.0.0",
+"karma-jasmine": "^1.1.2",
+"karma-jasmine-html-reporter": "^0.2.2",
"phantomjs-prebuilt": "^2.1.14",
-"protractor": "4.0.5",
-"ts-node": "1.2.1",
-"tslint": "3.13.0",
-"typescript": "~2.0.3",
-"@types/request": "2.0.3"
+"protractor": "^5.4.1",
+"ts-node": "~5.0.1",
+"tslint": "^5.11.0",
+"typescript": "~2.7.2"
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/0c4c622b/metron-interface/metron-config/pom.xml
--
diff --git a/metron-interface/metron-config/pom.xml
b/metron-interface/metron-config/pom.xml
index fd76447..c6ef5b5 100644
--- a/metron-interface/metron-config/pom.xml
+++ b/metron-interface/metron-config/pom.xml
@@ -63,7 +63,7 @@
generate-resources
-ng build
+npm run build
npm
@@ -72,13 +72,13 @@
-npm test
+test
+npm testCI
- npm
+npm
-test
- test
+run testCI
http://git-wip-us.apache.org/repos/asf/metron/blob/0c4c622b/metron-interface/metron-config/scripts/prepend_license_header.sh
--
diff --git a/metron-interface/metron-config/scripts/prepend_license_header.sh
b/metron-interface/metron-config/scripts/prepend_license_header.sh
index 1957cd6..6bf004f 100755
--- a/metron-interface/metron-config/scripts/prepend_license_header.sh
[52/52] [abbrv] metron git commit: Merge branch 'master' of http://github.com/apache/metron into feature/METRON-1090-stellar-assignment
Merge branch 'master' of http://github.com/apache/metron into feature/METRON-1090-stellar-assignment Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f548b191 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f548b191 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f548b191 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: f548b191b8348e14825fd9ec4bff3703193be51d Parents: 3df9498 3c4c069 Author: otto Authored: Sun Nov 18 13:11:19 2018 -0500 Committer: otto Committed: Sun Nov 18 13:11:19 2018 -0500 -- Upgrading.md|24 + dependencies_with_url.csv | 4 + .../committer-utils/metron-committer-common | 358 + dev-utilities/committer-utils/prepare-commit| 241 +- metron-analytics/metron-maas-common/pom.xml | 2 +- metron-deployment/Kerberos-manual-setup.md | 154 +- metron-deployment/development/README.md |34 + .../development/knox-demo-ldap.ldif | 101 + .../configuration/metron-enrichment-env.xml | 8 +- .../METRON/CURRENT/configuration/metron-env.xml | 9 - .../CURRENT/configuration/metron-rest-env.xml |10 +- .../configuration/metron-security-env.xml | 186 + .../common-services/METRON/CURRENT/metainfo.xml | 3 + .../package/files/metaalert_index.template | 4 +- .../CURRENT/package/scripts/metron_service.py | 2 - .../package/scripts/params/params_linux.py |30 +- .../CURRENT/package/scripts/rest_commands.py| 4 + .../METRON/CURRENT/package/templates/metron.j2 |16 + .../METRON/CURRENT/themes/metron_theme.json | 181 +- .../docker/rpm-docker/SPECS/metron.spec | 6 +- metron-interface/metron-alerts/cypress.json | 7 + .../metron-alerts/cypress/fixtures/config.json |23 + .../cypress/fixtures/pcap.page-01.json | 12383 +++ .../cypress/fixtures/pcap.status-00.json| 7 + .../cypress/fixtures/pcap.status-01.json| 7 + .../cypress/fixtures/pcap.status-02.json| 7 + .../metron-alerts/cypress/fixtures/search.json | 5647 + .../cypress/integration/pcap/pcap.spec.js | 228 + .../metron-alerts/package-lock.json | 1134 + metron-interface/metron-alerts/package.json |11 +- .../alert-details/alert-details.component.ts|10 +- .../alerts/alerts-list/alerts-list.component.ts | 9 +- .../table-view/table-view.component.spec.ts |13 +- .../table-view/table-view.component.ts |49 +- .../tree-view/tree-view.component.spec.ts |13 +- .../tree-view/tree-view.component.ts|20 +- .../alerts/meta-alerts/meta-alerts.component.ts | 2 - .../save-search/save-search.component.spec.ts | 4 +- .../alerts/save-search/save-search.component.ts |12 +- .../saved-searches.component.spec.ts| 5 +- .../saved-searches/saved-searches.component.ts |41 +- .../metron-alerts/src/app/app.component.html| 1 + .../metron-alerts/src/app/app.component.spec.ts | 4 + .../metron-alerts/src/app/app.module.ts |21 +- .../src/app/model/confirmation-type.ts |21 + .../metron-alerts/src/app/model/dialog-type.ts |21 + .../pcap-panel/pcap-panel.component.spec.ts | 9 + .../app/pcap/pcap-panel/pcap-panel.component.ts |68 +- .../src/app/pcap/service/pcap.service.spec.ts |13 +- .../src/app/pcap/service/pcap.service.ts|17 +- .../src/app/service/alerts.service.ts | 5 +- .../src/app/service/app-config.service.ts |41 + .../src/app/service/authentication.service.ts |10 +- .../src/app/service/column-names.service.ts | 4 - .../src/app/service/dialog.service.spec.ts |78 + .../src/app/service/dialog.service.ts |55 + .../src/app/service/global-config.service.ts| 5 +- .../src/app/service/meta-alert.service.ts |11 +- .../src/app/service/search.service.ts |11 +- .../src/app/service/update.service.ts | 9 +- .../src/app/shared/metron-dialog-box.ts |91 - .../metron-dialog/metron-dialog.component.html |40 + .../metron-dialog/metron-dialog.component.scss |20 + .../metron-dialog.component.spec.ts | 127 + .../metron-dialog/metron-dialog.component.ts|66 + .../metron-alerts/src/assets/app-config.json| 3 + metron-interface/metron-config/angular-cli.json |51 - metron-interface/metron-config/angular.json | 155 + metron-interface/metron-config/karma.conf.js|60 +- .../metron-config/package-lock.json | 18987 ++--- metron-interface/metron-config/package.json |70 +- metron-interface/metron-config/pom.xml |10
[03/52] [abbrv] metron git commit: METRON-1803: Integrate Cypress with Travis (tiborm via mmiklavc) closes apache/metron#1226
http://git-wip-us.apache.org/repos/asf/metron/blob/9b6260fd/metron-interface/metron-alerts/cypress/fixtures/pcap.status-00.json
--
diff --git
a/metron-interface/metron-alerts/cypress/fixtures/pcap.status-00.json
b/metron-interface/metron-alerts/cypress/fixtures/pcap.status-00.json
new file mode 100644
index 000..2a3cdcc
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress/fixtures/pcap.status-00.json
@@ -0,0 +1,7 @@
+{
+ "jobId":"job_1537878471649_0001",
+ "jobStatus":"RUNNING",
+ "description":"map: 0.0%, reduce: 0.0%",
+ "percentComplete":0.0,
+ "pageTotal":0
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/9b6260fd/metron-interface/metron-alerts/cypress/fixtures/pcap.status-01.json
--
diff --git
a/metron-interface/metron-alerts/cypress/fixtures/pcap.status-01.json
b/metron-interface/metron-alerts/cypress/fixtures/pcap.status-01.json
new file mode 100644
index 000..1505f71
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress/fixtures/pcap.status-01.json
@@ -0,0 +1,7 @@
+{
+ "jobId":"job_1537878471649_0001",
+ "jobStatus":"RUNNING",
+ "description":"map: 100.0%, reduce: 100.0%",
+ "percentComplete":75.0,
+ "pageTotal":0
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/9b6260fd/metron-interface/metron-alerts/cypress/fixtures/pcap.status-02.json
--
diff --git
a/metron-interface/metron-alerts/cypress/fixtures/pcap.status-02.json
b/metron-interface/metron-alerts/cypress/fixtures/pcap.status-02.json
new file mode 100644
index 000..662c27a
--- /dev/null
+++ b/metron-interface/metron-alerts/cypress/fixtures/pcap.status-02.json
@@ -0,0 +1,7 @@
+{
+ "jobId":"job_1537878471649_0001",
+ "jobStatus":"SUCCEEDED",
+ "description":"Job completed.",
+ "percentComplete":100.0,
+ "pageTotal":2
+}
\ No newline at end of file
[18/52] [abbrv] metron git commit: METRON-1749 Update Angular to latest release in Management UI (sardell via nickwallen) closes apache/metron#1217
http://git-wip-us.apache.org/repos/asf/metron/blob/0c4c622b/metron-interface/metron-config/src/app/shared/sample-data/sample-data.component.spec.ts
--
diff --git
a/metron-interface/metron-config/src/app/shared/sample-data/sample-data.component.spec.ts
b/metron-interface/metron-config/src/app/shared/sample-data/sample-data.component.spec.ts
index 5488209..b8cdf1f 100644
---
a/metron-interface/metron-config/src/app/shared/sample-data/sample-data.component.spec.ts
+++
b/metron-interface/metron-config/src/app/shared/sample-data/sample-data.component.spec.ts
@@ -15,17 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import {async, TestBed, ComponentFixture} from '@angular/core/testing';
-import {KafkaService} from '../../service/kafka.service';
-import {Observable} from 'rxjs/Observable';
-import {SampleDataComponent} from './sample-data.component';
-import {SharedModule} from '../shared.module';
-import '../../rxjs-operators';
+import { async, TestBed, ComponentFixture } from '@angular/core/testing';
+import { KafkaService } from '../../service/kafka.service';
+import { Observable, throwError } from 'rxjs';
+import { SampleDataComponent } from './sample-data.component';
+import { SharedModule } from '../shared.module';
class MockKafkaService {
_sample: string[];
- _sampleCounter: number = 0;
-
+ _sampleCounter = 0;
public setSample(sampleMessages: string[]): void {
this._sample = sampleMessages;
@@ -33,7 +31,6 @@ class MockKafkaService {
}
public sample(name: string): Observable {
-
if (this._sampleCounter < this._sample.length) {
return Observable.create(observer => {
observer.next(this._sample[this._sampleCounter++]);
@@ -41,7 +38,7 @@ class MockKafkaService {
});
}
-return Observable.throw('Error');
+return throwError('Error');
}
}
@@ -58,24 +55,21 @@ describe('SampleDataComponent', () => {
beforeEach(async(() => {
TestBed.configureTestingModule({
imports: [SharedModule],
- declarations: [ SampleDataComponent],
+ declarations: [SampleDataComponent],
providers: [
SampleDataComponent,
-{provide: KafkaService, useClass: MockKafkaService}
+{ provide: KafkaService, useClass: MockKafkaService }
]
});
-
fixture = TestBed.createComponent(SampleDataComponent);
sampleDataComponent = fixture.componentInstance;
-kafkaService = fixture.debugElement.injector.get(KafkaService);
-
+kafkaService = TestBed.get(KafkaService);
}));
it('can instantiate SampleDataComponent', async(() => {
expect(sampleDataComponent instanceof SampleDataComponent).toBe(true);
}));
-
it('should emmit messages', async(() => {
let expectedMessage;
let successCount = 0;
@@ -140,11 +134,9 @@ describe('SampleDataComponent', () => {
sampleDataComponent.getPreviousSample();
expect(successCount).toEqual(7);
expect(failureCount).toEqual(1);
-
}));
it('should emmit messages on blur', async(() => {
-
let expectedMessage;
let successCount = 0;
@@ -155,9 +147,10 @@ describe('SampleDataComponent', () => {
expect(message).toEqual(expectedMessage);
});
-
expectedMessage = 'This is a simple message';
-fixture.debugElement.nativeElement.querySelector('textarea').value =
expectedMessage;
+fixture.debugElement.nativeElement.querySelector(
+ 'textarea'
+).value = expectedMessage;
sampleDataComponent.onBlur();
expect(successCount).toEqual(1);
@@ -165,16 +158,16 @@ describe('SampleDataComponent', () => {
expect(sampleDataComponent.sampleData.length).toEqual(1);
expect(sampleDataComponent.sampleData[0]).toEqual(expectedMessage);
-
expectedMessage = '';
-fixture.debugElement.nativeElement.querySelector('textarea').value =
expectedMessage;
+fixture.debugElement.nativeElement.querySelector(
+ 'textarea'
+).value = expectedMessage;
sampleDataComponent.onBlur();
expect(successCount).toEqual(2);
expect(sampleDataComponent.sampleDataIndex).toEqual(0);
expect(sampleDataComponent.sampleData.length).toEqual(1);
-
expectedMessage = sampleMessages[0];
sampleDataComponent.getNextSample();
@@ -182,7 +175,5 @@ describe('SampleDataComponent', () => {
expect(sampleDataComponent.sampleDataIndex).toEqual(1);
expect(sampleDataComponent.sampleData.length).toEqual(2);
expect(sampleDataComponent.sampleData[1]).toEqual(sampleMessages[0]);
-
}));
-
});
http://git-wip-us.apache.org/repos/asf/metron/blob/0c4c622b/metron-interface/metron-config/src/app/util/httpUtil.ts
--
diff --git a/metron-interface/metron-config/src/app/util/httpUtil.ts
b/metron-interface/metron-config/src/app/util/httpUtil.ts
index dfcb61f..d8a21a5
[30/52] [abbrv] metron git commit: METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
--
diff --git
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
index 2c90b1e..cc6191c 100644
---
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
+++
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
@@ -18,6 +18,7 @@
package org.apache.metron.parsers.paloalto;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import org.apache.metron.parsers.AbstractParserConfigTest;
import org.json.simple.JSONObject;
@@ -25,6 +26,8 @@ import org.json.simple.parser.ParseException;
import org.junit.Before;
import org.junit.Test;
+import java.util.List;
+
public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
@Before
@@ -32,6 +35,221 @@ public class BasicPaloAltoFirewallParserTest extends
AbstractParserConfigTest {
parser = new BasicPaloAltoFirewallParser();
}
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testParseSystem61() throws ParseException {
+final String SYSTEM_61 = "1,2017/08/11
12:37:58,00898659,SYSTEM,general,1,2017/08/11
11:37:58,vsys1,eventId_test,object_test,Futureuse1_test,futureuse2_test,management,high,Description_test,1354,0x0";
+
+JSONObject actual = parser.parse(SYSTEM_61.getBytes()).get(0);
+
+JSONObject expected = new JSONObject();
+expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
+expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/08/11
12:37:58");
+expected.put(BasicPaloAltoFirewallParser.SerialNum, "00898659");
+expected.put(BasicPaloAltoFirewallParser.Type, "SYSTEM");
+expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "general");
+expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1");
+expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/08/11
11:37:58");
+expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1");
+expected.put(BasicPaloAltoFirewallParser.EventId, "eventId_test");
+expected.put(BasicPaloAltoFirewallParser.Object, "object_test");
+expected.put(BasicPaloAltoFirewallParser.Module, "management");
+expected.put(BasicPaloAltoFirewallParser.Severity, "high");
+expected.put(BasicPaloAltoFirewallParser.Description, "Description_test");
+expected.put(BasicPaloAltoFirewallParser.Seqno, "1354");
+expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0");
+expected.put(BasicPaloAltoFirewallParser.ParserVersion, 61);
+expected.put("original_string", SYSTEM_61);
+expected.put("timestamp", actual.get("timestamp"));
+
+assertEquals(expected, actual);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testParseSystem80() throws ParseException {
+final String SYSTEM_80 = "1,2017/08/11
12:37:58,00898659,SYSTEM,general,1,2017/08/11
11:37:58,vsys1,eventId_test,object_test,Futureuse1_test,futureuse2_test,management,high,Description_test,1354,0x0,12,34,45,0,virSys1,dev-something200-01";
+
+JSONObject actual = parser.parse(SYSTEM_80.getBytes()).get(0);
+
+JSONObject expected = new JSONObject();
+expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
+expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/08/11
12:37:58");
+expected.put(BasicPaloAltoFirewallParser.SerialNum, "00898659");
+expected.put(BasicPaloAltoFirewallParser.Type, "SYSTEM");
+expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "general");
+expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1");
+expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/08/11
11:37:58");
+expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1");
+expected.put(BasicPaloAltoFirewallParser.EventId, "eventId_test");
+expected.put(BasicPaloAltoFirewallParser.Object, "object_test");
+expected.put(BasicPaloAltoFirewallParser.Module, "management");
+expected.put(BasicPaloAltoFirewallParser.Severity, "high");
+expected.put(BasicPaloAltoFirewallParser.Description, "Description_test");
+expected.put(BasicPaloAltoFirewallParser.Seqno, "1354");
+expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0");
+expected.put(BasicPaloAltoFirewallParser.DGH1, "12");
+expected.put(BasicPaloAltoFirewallParser.DGH2, "34");
+expected.put(BasicPaloAltoFirewallParser.DGH3, "45");
+expected.put(BasicPaloAltoFirewallParser.DGH4, "0");
+expected.put(BasicPaloAltoFirewallParser.VSYSName, "virSys1");
+
metron git commit: METRON-1847 Create reusable script with functions from prepare-commit (ottobackwards) closes apache/metron#1248
Repository: metron
Updated Branches:
refs/heads/master 3e73391ed -> 42068d951
METRON-1847 Create reusable script with functions from prepare-commit
(ottobackwards) closes apache/metron#1248
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/42068d95
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/42068d95
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/42068d95
Branch: refs/heads/master
Commit: 42068d95192e7346bf0179399193a6afe01d9925
Parents: 3e73391
Author: ottobackwards
Authored: Thu Nov 8 22:14:38 2018 -0500
Committer: otto
Committed: Thu Nov 8 22:14:38 2018 -0500
--
.../committer-utils/metron-committer-common | 357 +++
dev-utilities/committer-utils/prepare-commit| 241 +
2 files changed, 372 insertions(+), 226 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/42068d95/dev-utilities/committer-utils/metron-committer-common
--
diff --git a/dev-utilities/committer-utils/metron-committer-common
b/dev-utilities/committer-utils/metron-committer-common
new file mode 100644
index 000..22e19a2
--- /dev/null
+++ b/dev-utilities/committer-utils/metron-committer-common
@@ -0,0 +1,357 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# common and unlikely to change vars
+#
+
+# the upstream apache git repo for apache metron
+METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron.git;
+# the upstream apache git repo for apache metron bro plugin kafka
+BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git;
+# the common configuration file with the committer info
+CONFIG_FILE=~/.metron-prepare-commit
+
+GITHUB_REMOTE="origin"
+BASE_BRANCH=master
+
+# other var setup by these functions
+PR=
+WORK=
+ORIGIN=
+UPSTREAM=
+PR_BRANCH=
+USER=
+EMAIL=
+JIRA=
+DESC=
+APACHE_NAME=
+APACHE_EMAIL=
+GITHUB_NAME=
+
+#
+# Initialize the variables from the default configuration file, if it exists
+#
+function init_configuration {
+# does a config file already exist?
+echo "$CONFIG_FILE"
+if [ -f ${CONFIG_FILE} ]; then
+#shellcheck source=/dev/null
+source ${CONFIG_FILE}
+echo " ...using settings from $CONFIG_FILE"
+fi
+}
+
+
+#
+# Initialize the committer variables if they are not provided through the
configuration file.
+# If it is not present, it will be written out for the next time
+#
+function init_committer_info {
+# github account of committer (you)
+if [ -z "$GITHUB_NAME" ]; then
+read -p " your github username [$GITHUB_NAME]: " INPUT
+[ -n "$INPUT" ] && GITHUB_NAME=${INPUT}
+
+# write setting to config file
+echo "GITHUB_NAME=$GITHUB_NAME" >> ${CONFIG_FILE}
+fi
+
+# apache id of committer (you)
+if [ -z "$APACHE_NAME" ]; then
+ read -p " your apache userid [$APACHE_NAME]: " INPUT
+ [ -n "$INPUT" ] && APACHE_NAME=${INPUT}
+
+ # write setting to config file
+ echo "APACHE_NAME=$APACHE_NAME" >> ${CONFIG_FILE}
+fi
+
+# apache email addr of committer (you)
+if [ -z "$APACHE_EMAIL" ]; then
+ APACHE_EMAIL=${APACHE_NAME}@apache.org
+ read -p " your apache email [$APACHE_EMAIL]: " INPUT
+ [ -n "$INPUT" ] && APACHE_EMAIL=${INPUT}
+
+ # write setting to config file, so it is not needed next time
+ echo "APACHE_EMAIL=$APACHE_EMAIL" >> ${CONFIG_FILE}
+fi
+}
+
+#
+# Provide the user with a choice of the metron or bro repositories
+#
+function choose_metron_or_bro_repo {
+# which repo? metron or metron-bro-plugin-kafka
+echo "[1] metron"
+echo "[2] metron-bro-plugin-kafka"
+read -p " which repo? [1]: " INPUT
+case "
metron git commit: METRON-1858 BasicFireEyeParser check style cleanup and optimization (ottobackwards) closes apache/metron#1255
Repository: metron
Updated Branches:
refs/heads/master c6b1b3ccd -> b5712af5b
METRON-1858 BasicFireEyeParser check style cleanup and optimization
(ottobackwards) closes apache/metron#1255
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b5712af5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b5712af5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b5712af5
Branch: refs/heads/master
Commit: b5712af5b78ff8c6e5e3d3f3c68552204fa7d877
Parents: c6b1b3c
Author: ottobackwards
Authored: Thu Nov 8 14:14:03 2018 -0500
Committer: otto
Committed: Thu Nov 8 14:14:03 2018 -0500
--
.../parsers/fireeye/BasicFireEyeParser.java | 347 +--
1 file changed, 158 insertions(+), 189 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/b5712af5/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
--
diff --git
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
index 489eb00..1cf9fb7 100644
---
a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
+++
b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,18 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.metron.parsers.fireeye;
import com.google.common.base.Joiner;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.metron.parsers.utils.ParserUtils;
-import org.apache.metron.parsers.BasicParser;
-import org.json.simple.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -34,189 +30,162 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.parsers.BasicParser;
+import org.apache.metron.parsers.utils.ParserUtils;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-public class BasicFireEyeParser extends BasicParser {
-
- private static final long serialVersionUID = 6328907550159134550L;
- protected static final Logger LOG = LoggerFactory
- .getLogger(BasicFireEyeParser.class);
-
-
- String tsRegex
="([a-zA-Z]{3})\\s+(\\d+)\\s+(\\d+\\:\\d+\\:\\d+)\\s+(\\d+\\.\\d+\\.\\d+\\.\\d+)";
-
-
- Pattern tsPattern = Pattern.compile(tsRegex);
- // private transient static MetronGrok grok;
- // private transient static InputStream pattern_url;
-
- public BasicFireEyeParser() throws Exception {
- // pattern_url =
getClass().getClassLoader().getResourceAsStream(
- // "patterns/fireeye");
- //
- // File file = ParserUtils.stream2file(pattern_url);
- // grok = MetronGrok.create(file.getPath());
- //
- // grok.compile("%{FIREEYE_BASE}");
- }
-
- @Override
- public void configure(Map parserConfig) {
-
- }
-
- @Override
- public void init() {
-
- }
-
- @Override
- public List parse(byte[] raw_message) {
- String toParse = "";
- List messages = new ArrayList<>();
- try {
-
- toParse = new String(raw_message, "UTF-8");
-
- // String[] mTokens = toParse.split(" ");
-
- String positveIntPattern = "<[1-9][0-9]*>";
- Pattern p = Pattern.compile(positveIntPattern);
- Matcher m = p.matcher(toParse);
-
- String delimiter = "";
-
- while (m.find()) {
- delimiter = m.group();
-
- }
-
- if (!StringUtils.isBlank(delimiter)) {
- Stri
metron git commit: METRON-1864 Stellar date format test fails after daylight saving (ottobackwards) closes apache/metron#1258
Repository: metron
Updated Branches:
refs/heads/master 7bba8be0c -> c6b1b3ccd
METRON-1864 Stellar date format test fails after daylight saving
(ottobackwards) closes apache/metron#1258
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c6b1b3cc
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c6b1b3cc
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c6b1b3cc
Branch: refs/heads/master
Commit: c6b1b3ccd7f937342c5723ca0a952a4900efc5cf
Parents: 7bba8be
Author: ottobackwards
Authored: Thu Nov 8 13:51:38 2018 -0500
Committer: otto
Committed: Thu Nov 8 13:51:38 2018 -0500
--
.../metron/stellar/dsl/functions/DateFunctionsTest.java | 12 ++--
1 file changed, 10 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/c6b1b3cc/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
--
diff --git
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
index 959e7ea..48b2995 100644
---
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
+++
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/DateFunctionsTest.java
@@ -28,6 +28,8 @@ import org.apache.metron.stellar.dsl.StellarFunctions;
import org.junit.Before;
import org.junit.Test;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Map;
@@ -233,10 +235,16 @@ public class DateFunctionsTest {
assertEquals("Thu Aug 25 2016 08:27:10 EST", result);
}
+ /**
+ * Test that the String returned is formatted as specified.
+ * LocalDate.parse will throw if it is not.
+ * @throws Exception
+ */
@Test
- public void testDateFormatDefault() {
+ public void testDateFormatDefault() throws Exception {
Object result = run("DATE_FORMAT('EEE MMM dd hh:mm:ss ')");
-
assertTrue(result.toString().endsWith(TimeZone.getDefault().getDisplayName(true,
1)));
+DateTimeFormatter formatter = DateTimeFormatter.ofPattern("EEE MMM dd
hh:mm:ss ");
+LocalDate.parse(result.toString(), formatter);
}
@Test
[3/3] metron git commit: METRON-1563 Initial Feature Branch Work (ottobackwards) closes apache/metron#1014
METRON-1563 Initial Feature Branch Work (ottobackwards) closes
apache/metron#1014
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3df94987
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3df94987
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3df94987
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 3df949877d0da842488e3440563d2350bfe64b96
Parents: 0e037ed
Author: ottobackwards
Authored: Fri Oct 26 09:02:27 2018 -0400
Committer: otto
Committed: Fri Oct 26 09:02:27 2018 -0400
--
.../StellarEnrichmentConfigTest.java| 12 +-
.../metron/pcap/filter/PcapFieldResolver.java |2 +
metron-stellar/stellar-common/README.md | 11 +-
.../metron/stellar/common/generated/Stellar.g4 | 32 +-
.../stellar-common/src/main/java/Stellar.tokens | 130 +-
.../src/main/java/StellarLexer.tokens | 130 +-
.../stellar/common/BaseStellarProcessor.java|2 +-
.../metron/stellar/common/LambdaExpression.java | 22 +-
.../metron/stellar/common/StellarCompiler.java | 211 +++
.../common/generated/StellarBaseListener.java | 156 +++
.../stellar/common/generated/StellarLexer.java | 430 +++---
.../common/generated/StellarListener.java | 156 +++
.../stellar/common/generated/StellarParser.java | 1306 --
.../common/utils/StellarProcessorUtils.java |1 +
.../stellar/dsl/DefaultVariableResolver.java| 73 +-
.../metron/stellar/dsl/MapVariableResolver.java |5 +
.../metron/stellar/dsl/VariableResolver.java| 27 +
.../stellar/dsl/functions/BasicStellarTest.java | 638 +
18 files changed, 2556 insertions(+), 788 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/3df94987/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
--
diff --git
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
index 32fb00e..5ae 100644
---
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
+++
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java
@@ -81,7 +81,7 @@ public class StellarEnrichmentConfigTest extends
StellarEnrichmentTest {
List splits = Configs.STELLAR.splitByFields(message, null, x
-> null, handler );
Assert.assertEquals(1, splits.size());
Map split = (Map) splits.get(0).get("");
- Assert.assertEquals(3, split.size());
+ Assert.assertTrue(split.size() == 3 || split.size() == 5 || split.size()
== 6);
Assert.assertEquals("stellar_test", split.get("source.type"));
Assert.assertEquals("foo", split.get("string"));
Assert.assertNull(split.get("stmt1"));
@@ -111,13 +111,13 @@ public class StellarEnrichmentConfigTest extends
StellarEnrichmentTest {
Assert.assertEquals(2, splits.size());
{
Map split = (Map)
splits.get(0).get("group1");
-Assert.assertEquals(2, split.size());
+Assert.assertTrue(split.size() == 2 || split.size() == 3);
Assert.assertEquals("stellar_test", split.get("source.type"));
Assert.assertNull(split.get("stmt1"));
}
{
Map split = (Map)
splits.get(1).get("group2");
-Assert.assertEquals(1, split.size());
+Assert.assertTrue(split.size() == 1 | split.size() == 2 ||
split.size() == 3);
Assert.assertEquals("foo", split.get("string"));
}
}
@@ -148,18 +148,18 @@ public class StellarEnrichmentConfigTest extends
StellarEnrichmentTest {
Assert.assertEquals(3, splits.size());
{
Map split = (Map)
splits.get(0).get("group1");
-Assert.assertEquals(2, split.size());
+Assert.assertTrue(split.size() == 2 || split.size() == 3);
Assert.assertEquals("stellar_test", split.get("source.type"));
Assert.assertNull(split.get("stmt1"));
}
{
Map split = (Map)
splits.get(1).get("group2");
-Assert.assertEquals(1, split.size());
+Assert.assertTrue(split.size() == 1 || split.size() == 2);
Assert.assertEquals("foo", split.get("string"));
}
{
Map split = (Map)
splits.get(2).get("");
-Assert.assertEquals(1, split.size());
+A
[1/3] metron git commit: METRON-1563 Initial Feature Branch Work (ottobackwards) closes apache/metron#1014
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1090-stellar-assignment 0e037edad -> 3df949877
http://git-wip-us.apache.org/repos/asf/metron/blob/3df94987/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/DefaultVariableResolver.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/DefaultVariableResolver.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/DefaultVariableResolver.java
index fc2c2b7..1ac34d5 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/DefaultVariableResolver.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/DefaultVariableResolver.java
@@ -1,44 +1,75 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional
information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache
License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the
License. You may obtain
+ * a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express
+ * or implied. See the License for the specific language governing permissions
and limitations under
+ * the License.
*/
package org.apache.metron.stellar.dsl;
+import java.util.function.BiConsumer;
import java.util.function.Function;
-public class DefaultVariableResolver implements VariableResolver{
- Function resolveFunc;
- Function existsFunc;
+/**
+ * Simple VariableResolver implemenation using passed Functions
+ * for implementation.
+ *
+ * Support for updates is optional
+ */
+public class DefaultVariableResolver implements VariableResolver {
- public DefaultVariableResolver(Function resolveFunc,
Function existsFunc){
+ private Function resolveFunc;
+ private Function existsFunc;
+ private BiConsumer updateFunc;
+
+ /**
+ * DefaultVariableResolver without support for updates
+ * @param resolveFunc
+ * @param existsFunc
+ */
+ public DefaultVariableResolver(Function resolveFunc,
+ Function existsFunc) {
+this(resolveFunc, existsFunc, null);
+ }
+
+ /**
+ * DefaultVariableResolver with full support for updates
+ * @param resolveFunc
+ * @param existsFunc
+ * @param updateFunc
+ */
+ public DefaultVariableResolver(Function resolveFunc,
+ Function existsFunc, BiConsumer
updateFunc) {
this.resolveFunc = resolveFunc;
this.existsFunc = existsFunc;
+this.updateFunc = updateFunc;
}
+
@Override
public Object resolve(String variable) {
-return resolveFunc.apply(variable);
+return resolveFunc == null? null : resolveFunc.apply(variable);
}
@Override
public boolean exists(String variable) {
-return existsFunc.apply(variable);
+return existsFunc == null? false : existsFunc.apply(variable);
}
- public static DefaultVariableResolver NULL_RESOLVER() {
-return new DefaultVariableResolver(x -> null, x -> false);
+ @Override
+ public void update(String variable, Object value) {
+if (updateFunc != null) {
+ updateFunc.accept(variable, value);
+}
}
+
+ public static DefaultVariableResolver NULL_RESOLVER = new
DefaultVariableResolver(x -> null,
+ x -> false, null);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3df94987/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java
index 872211d..e057975 100644
---
[2/3] metron git commit: METRON-1563 Initial Feature Branch Work (ottobackwards) closes apache/metron#1014
http://git-wip-us.apache.org/repos/asf/metron/blob/3df94987/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarParser.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarParser.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarParser.java
index 3bd4ad1..3fda80b 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarParser.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/generated/StellarParser.java
@@ -38,48 +38,53 @@ public class StellarParser extends Parser {
new PredictionContextCache();
public static final int
IN=1, LAMBDA_OP=2, DOUBLE_QUOTE=3, SINGLE_QUOTE=4, COMMA=5,
PERIOD=6,
- AND=7, OR=8, NOT=9, TRUE=10, FALSE=11, EQ=12, NEQ=13, LT=14,
LTE=15, GT=16,
- GTE=17, QUESTION=18, COLON=19, IF=20, THEN=21, ELSE=22,
NULL=23, NAN=24,
- MATCH=25, DEFAULT=26, MATCH_ACTION=27, MINUS=28, PLUS=29,
DIV=30, MUL=31,
- LBRACE=32, RBRACE=33, LBRACKET=34, RBRACKET=35, LPAREN=36,
RPAREN=37,
- NIN=38, EXISTS=39, EXPONENT=40, INT_LITERAL=41,
DOUBLE_LITERAL=42, FLOAT_LITERAL=43,
- LONG_LITERAL=44, IDENTIFIER=45, STRING_LITERAL=46, COMMENT=47,
WS=48;
+ AND=7, OR=8, NOT=9, TRUE=10, FALSE=11, ASSIGN=12,
COLON_ASSIGN=13, PLUSASSIGN=14,
+ MINUSASSIGN=15, DIVIDEASSIGN=16, MULTASSIGN=17, EQ=18, NEQ=19,
LT=20,
+ LTE=21, GT=22, GTE=23, QUESTION=24, COLON=25, IF=26, THEN=27,
ELSE=28,
+ NULL=29, NAN=30, MATCH=31, DEFAULT=32, MATCH_ACTION=33,
MINUS=34, MINUSMINUS=35,
+ PLUS=36, PLUSPLUS=37, DIV=38, MUL=39, LBRACE=40, RBRACE=41,
LBRACKET=42,
+ RBRACKET=43, LPAREN=44, RPAREN=45, NIN=46, EXISTS=47,
EXPONENT=48, INT_LITERAL=49,
+ DOUBLE_LITERAL=50, FLOAT_LITERAL=51, LONG_LITERAL=52,
IDENTIFIER=53, STRING_LITERAL=54,
+ COMMENT=55, WS=56;
public static final int
- RULE_transformation = 0, RULE_transformation_expr = 1,
RULE_if_expr = 2,
- RULE_then_expr = 3, RULE_else_expr = 4, RULE_conditional_expr =
5, RULE_logical_expr = 6,
- RULE_b_expr = 7, RULE_in_expr = 8, RULE_comparison_expr = 9,
RULE_transformation_entity = 10,
- RULE_comp_operator = 11, RULE_func_args = 12, RULE_op_list =
13, RULE_list_entity = 14,
- RULE_kv_list = 15, RULE_map_entity = 16, RULE_arithmetic_expr =
17, RULE_arithmetic_expr_mul = 18,
- RULE_functions = 19, RULE_arithmetic_operands = 20,
RULE_identifier_operand = 21,
- RULE_default_operand = 22, RULE_lambda_without_args = 23,
RULE_lambda_with_args = 24,
- RULE_lambda_variables = 25, RULE_single_lambda_variable = 26,
RULE_lambda_variable = 27,
- RULE_match_expr = 28, RULE_match_clauses = 29,
RULE_match_clause = 30,
- RULE_match_clause_action = 31, RULE_match_clause_check = 32;
+ RULE_transformation = 0, RULE_transformation_expr = 1,
RULE_assign_expr = 2,
+ RULE_pre_expr = 3, RULE_post_expr = 4, RULE_if_expr = 5,
RULE_then_expr = 6,
+ RULE_else_expr = 7, RULE_conditional_expr = 8,
RULE_logical_expr = 9,
+ RULE_b_expr = 10, RULE_in_expr = 11, RULE_comparison_expr = 12,
RULE_transformation_entity = 13,
+ RULE_comp_operator = 14, RULE_func_args = 15, RULE_op_list =
16, RULE_list_entity = 17,
+ RULE_kv_list = 18, RULE_map_entity = 19, RULE_arithmetic_expr =
20, RULE_arithmetic_expr_mul = 21,
+ RULE_functions = 22, RULE_arithmetic_operands = 23,
RULE_identifier_operand = 24,
+ RULE_default_operand = 25, RULE_lambda_without_args = 26,
RULE_lambda_with_args = 27,
+ RULE_lambda_variables = 28, RULE_single_lambda_variable = 29,
RULE_lambda_variable = 30,
+ RULE_match_expr = 31, RULE_match_clauses = 32,
RULE_match_clause = 33,
+ RULE_match_clause_action = 34, RULE_match_clause_check = 35;
public static final String[] ruleNames = {
- "transformation", "transformation_expr", "if_expr",
"then_expr", "else_expr",
- "conditional_expr", "logical_expr", "b_expr", "in_expr",
"comparison_expr",
- "transformation_entity", "comp_operator", "func_args",
"op_list", "list_entity",
- "kv_list", "map_entity", "arithmetic_expr",
"arithmetic_expr_mul", "functions",
- "arithmetic_operands", "identifier_operand", "default_operand",
"lambda_without_args",
- "lambda_with_args", "lambda_variables",
"single_lambda_variable", "lambda_variable",
[43/51] [abbrv] metron git commit: METRON-1681 Decouple the ParserBolt from the Parse execution logic (merrimanr) closes apache/metron#1213
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
--
diff --git
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
new file mode 100644
index 000..5f05b24
--- /dev/null
+++
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
@@ -0,0 +1,390 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.error.MetronError;
+import org.apache.metron.common.message.metadata.RawMessage;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.parsers.filters.Filters;
+import org.apache.metron.parsers.filters.StellarFilter;
+import org.apache.metron.parsers.interfaces.MessageFilter;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.parsers.topology.ParserComponent;
+import org.apache.metron.parsers.ParserRunnerImpl.ProcessResult;
+import org.apache.metron.stellar.dsl.Context;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ParserRunnerImpl.class, ReflectionUtils.class, Filters.class})
+public class ParserRunnerImplTest {
+
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ /**
+ {
+ "fieldValidations" : [
+ {
+ "input" : [ "ip_src_addr", "ip_dst_addr"],
+ "validation" : "IP"
+ }
+ ]
+ }
+ */
+ @Multiline
+ private String globalConfigString;
+
+ /**
+ {
+ "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
+ "filterClassName":"org.apache.metron.parsers.filters.StellarFilter",
+ "sensorTopic":"bro",
+ "parserConfig": {
+ "field": "value"
+ },
+ "fieldTransformations" : [
+ {
+ "input" : "field1",
+ "transformation" : "REMOVE"
+ }
+ ]
+ }
+ */
+ @Multiline
+ private String broConfigString;
+
+ /**
+ {
+ "parserClassName":"org.apache.metron.parsers.snort.BasicSnortParser",
+ "sensorTopic":"snort",
+ "parserConfig": {}
+ }
+ */
+ @Multiline
+ private String snortConfigString;
+
+ private ParserConfigurations parserConfigurations;
+ private MessageParser broParser;
+ private MessageParser snortParser;
+ private MessageFilter stellarFilter;
+ private ParserRunnerImpl parserRunner;
+
+
+ @Before
+ public void setup() throws IOException {
+parserConfigurations = new ParserConfigurations();
+SensorParserConfig broConfig =
SensorParserConfig.fromBytes(broConfigString.getBytes());
+SensorParserConfig snortConfig =
SensorParserConfig.fromBytes(snortConfigString.getBytes());
+
[48/51] [abbrv] metron git commit: METRON-1833: Management UI incorrectly displaying sensor topology latency units as seconds instead of millis (mmiklavc via mmiklavc) closes apache/metron#1241
METRON-1833: Management UI incorrectly displaying sensor topology latency units
as seconds instead of millis (mmiklavc via mmiklavc) closes apache/metron#1241
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0c0602c7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0c0602c7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0c0602c7
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 0c0602c75d9660717920a547bcb9e76d58b6571e
Parents: d44a392
Author: mmiklavc
Authored: Mon Oct 22 12:54:26 2018 -0600
Committer: Michael Miklavcic
Committed: Mon Oct 22 12:54:26 2018 -0600
--
.../sensor-parser-config-readonly.component.ts | 2 +-
.../sensor-parser-list/sensor-parser-list.component.spec.ts| 2 +-
.../app/sensors/sensor-parser-list/sensor-parser-list.component.ts | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/0c0602c7/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
--
diff --git
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
index 5db6d45..7d41003 100644
---
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
+++
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.component.ts
@@ -156,7 +156,7 @@ export class SensorParserConfigReadonlyComponent implements
OnInit {
getTopologyStatus(key: string): string {
if (key === 'latency') {
- return this.topologyStatus.latency >= 0 ? (this.topologyStatus.latency +
's') : '-';
+ return this.topologyStatus.latency >= 0 ? (this.topologyStatus.latency +
'ms') : '-';
} else if (key === 'throughput') {
return this.topologyStatus.throughput >= 0 ?
((Math.round(this.topologyStatus.throughput * 100) / 100) + 'kb/s') : '-';
} else if (key === 'emitted') {
http://git-wip-us.apache.org/repos/asf/metron/blob/0c0602c7/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
--
diff --git
a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
index 205d885..fb2a175 100644
---
a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
+++
b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.spec.ts
@@ -720,7 +720,7 @@ describe('Component: SensorParserList', () => {
component.updateSensorStatus();
expect(component.sensors[0]['status']).toEqual('Running');
-expect(component.sensors[0]['latency']).toEqual('10s');
+expect(component.sensors[0]['latency']).toEqual('10ms');
expect(component.sensors[0]['throughput']).toEqual('23kb/s');
component.sensorsStatus[0].status = 'KILLED';
http://git-wip-us.apache.org/repos/asf/metron/blob/0c0602c7/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
--
diff --git
a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
index 1129914..2694ab4 100644
---
a/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
+++
b/metron-interface/metron-config/src/app/sensors/sensor-parser-list/sensor-parser-list.component.ts
@@ -163,7 +163,7 @@ export class SensorParserListComponent implements OnInit {
sensor['status'] = 'Stopped';
}
-sensor['latency'] = status && status.status === 'ACTIVE' ?
(status.latency + 's') : '-';
+sensor['latency'] = status && status.status === 'ACTIVE' ?
(status.latency + 'ms') : '-';
sensor['throughput'] = status && status.status === 'ACTIVE' ?
(Math.round(status.throughput * 100) / 100) + 'kb/s' : '-';
}
}
[46/51] [abbrv] metron git commit: METRON-1831 Project Version Substitution Not Working (nickwallen) closes apache/metron#1243
METRON-1831 Project Version Substitution Not Working (nickwallen) closes
apache/metron#1243
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2531c3e1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2531c3e1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2531c3e1
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 2531c3e19e6c2887a2242da830d7178e6973b8ab
Parents: bf32fef
Author: nickwallen
Authored: Sat Oct 20 11:48:38 2018 -0400
Committer: nickallen
Committed: Sat Oct 20 11:48:38 2018 -0400
--
metron-platform/metron-common/src/main/scripts/stellar | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/2531c3e1/metron-platform/metron-common/src/main/scripts/stellar
--
diff --git a/metron-platform/metron-common/src/main/scripts/stellar
b/metron-platform/metron-common/src/main/scripts/stellar
index 34097f2..690a9f0 100644
--- a/metron-platform/metron-common/src/main/scripts/stellar
+++ b/metron-platform/metron-common/src/main/scripts/stellar
@@ -33,8 +33,9 @@ if [ -f "$METRON_SYSCONFIG" ]; then
source $METRON_SYSCONFIG
fi
-export METRON_VERSION="${METRON_VERSION:-${project.version}}"
-export METRON_HOME="${METRON_HOME:-/usr/metron/$METRON_VERSION}"
+# treat unset vars as an error; METRON_HOME
+set -u
+
export HBASE_CONFIGS=$(hbase classpath)
export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar)
export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar)
[38/51] [abbrv] metron git commit: METRON-1799 Remove outdated bylaws from site. (justinleet) closes apache/metron#1216
METRON-1799 Remove outdated bylaws from site. (justinleet) closes apache/metron#1216 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9b671005 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9b671005 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9b671005 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 9b6710053894d8a39880cb8157a0e603ed542cb7 Parents: f153375 Author: justinleet Authored: Thu Oct 11 08:41:23 2018 -0400 Committer: leet Committed: Thu Oct 11 08:41:23 2018 -0400 -- site/develop/bylaws.md | 281 site/develop/coding.md | 34 -- site/develop/index.md | 61 -- 3 files changed, 376 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/9b671005/site/develop/bylaws.md -- diff --git a/site/develop/bylaws.md b/site/develop/bylaws.md deleted file mode 100644 index a8fc5fb..000 --- a/site/develop/bylaws.md +++ /dev/null @@ -1,281 +0,0 @@ -layout: page -title: Apache Metron Bylaws - -## Introduction - -This document defines the bylaws under which the Apache Metron project -operates. It defines the roles and responsibilities of the project, -who may vote, how voting works, how conflicts are resolved, etc. - -Metron is a project of the Apache Software Foundation (ASF) and the foundation -holds the trademark on the name "Metron" and copyright on the combined -code base. The [Apache Foundation -FAQ](http://www.apache.org/foundation/faq.html) and -[How-It-Works](http://www.apache.org/foundation/how-it-works.html) -explain the operation and background of the foundation. - -Apache has a [code of -conduct](http://www.apache.org/foundation/policies/conduct.html) that -it expects its members to follow. In particular: - -* Be **open and welcoming**. It is important that we grow and - encourage the community of users and developers for our project. - -* Be **collaborative**. Working together on the open mailing lists and - bug database to make decisions helps the project grow. - -* Be **respectful** of others. Everyone is volunteering their time and - efforts to work on this project. Please be respectful of everyone - and their views. - -Metron is typical of Apache projects in that it operates under a set of -principles, known collectively as the "Apache Way". If you are new to -Apache development, please refer to -[this](http://www.apache.org/foundation/how-it-works.html) for more -information on how Apache projects operate. - -## Roles and Responsibilities - -Apache projects define a set of roles with associated rights and -responsibilities. These roles govern what tasks an individual may -perform within the project. The roles are defined in the following -sections. - -### Users - -The most important participants in the project are people who use our -software. The majority of our developers start out as users and guide -their development efforts from the user's perspective. Users -contribute to the Apache projects by providing feedback to developers -in the form of bug reports and feature suggestions. As well, users -participate in the Apache community by helping other users on mailing -lists and user support forums. - -### Contributors - -Contributors include all of the volunteers who donate time, code, -documentation, -or resources to the Metron Project. A contributor that makes sustained, -welcome contributions to the project may be invited to become a -committer, though the exact timing of such invitations depends on many -factors. - -### Committers - -The project's committers are responsible for the project's technical -management. Committers have the right to commit to the project's git -repository. Committers may cast binding votes on any technical -discussion. - -Committer access is by invitation only and must be approved by -consensus approval of the active Project Management Committee (PMC) -members. - -If a committer wishes to leave the project or does not contribute to -the project in any form for six months, the PMC may make them emeritus. -Emeritus committers lose their ability to commit code or cast binding -votes. An emeritus committer may -request reinstatement of commit access from the PMC. Such -reinstatement is subject to consensus approval of active PMC members. - -All Apache committers are required to have a signed [Individual -Contributor License -Agreement](https://www.apache.org/licenses/icla.txt) (ICLA) on file -with the Apache Software Foundation. There is a [Committer -FAQ](http://www.apache.org/dev/committers.html) which provides more -details on the requirements for Committers. - -A committer who makes a -sustained contribution to the project may be invited to become a -member
[37/51] [abbrv] metron git commit: METRON-1769 Script creation of a release candidate (justinleet) closes apache/metron#1188
METRON-1769 Script creation of a release candidate (justinleet) closes apache/metron#1188 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f153375f Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f153375f Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f153375f Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: f153375fad65e75a10ce490b4a5f29f7e82f70dd Parents: 3d923cd Author: justinleet Authored: Wed Oct 10 15:41:22 2018 -0400 Committer: leet Committed: Wed Oct 10 15:41:22 2018 -0400 -- dev-utilities/release-utils/README.md | 129 +++ dev-utilities/release-utils/metron-rc-check | 6 +- .../release-utils/prepare-release-candidate | 337 +++ 3 files changed, 469 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/f153375f/dev-utilities/release-utils/README.md -- diff --git a/dev-utilities/release-utils/README.md b/dev-utilities/release-utils/README.md new file mode 100644 index 000..d1fd816 --- /dev/null +++ b/dev-utilities/release-utils/README.md @@ -0,0 +1,129 @@ + + +# Release Tools + +This project contains tools to assist Apache Metron project committers. + +## Prepare Release Candidate + +This script automates the process of creating a release candidate from `apache/metron` or `apache/metron-bro-plugin-kafka`. The script will prompt for various information necessary. Ensure your signing key is setup per [Release Signing](https://www.apache.org/dev/release-signing.html) and [Apache GnuPGP Instructions](https://www.apache.org/dev/openpgp.html#gnupg) + +When prompted the `[value in brackets]` is used by default. To accept the default, simply press `enter`. If you would like to change the default, type it in and hit `enter` when done. + +In the following example, enter the appropriate information + +1. Execute the script. + +The first time the script is run, you will be prompted for additional information including your Apache username and Apache email. These values are persisted in `~/.metron-prepare-release-candidate`. Subsequent executions of the script will retrieve these values, rather than prompting you again for them. + +``` +$ ./prepare-release-candidate + your apache userid []: leet + your apache email [l...@apache.org]: +``` + +1. Select a repository we're creating an RC for. + +``` +[1] metron +[2] metron-bro-plugin-kafka + which repo? [1]: 1 +``` + +1. Enter the current version number. This will be the base for the CHANGES file + +``` + current version: 0.6.0 +``` + +1. Enter the version being built. + +``` + version being built: 0.6.1 +``` + +1. Enter the current RC number + +``` + release candidate number: 1 +``` + +1. Enter the branch we're releasing from. In most cases, this will be master, but for maintenance releases it can be another branch. + +``` + base revision branch or hash for release candidate [master]: +``` + +1. Enter the signing key id. + +``` + signing key id in 8-byte format (e.g. BADDCAFEDEADBEEF): +``` + +1. Enter if this is a practice run. In a practice run, nothing is pushed to SVN, but everything is setup and built otherwise. + +``` + do a live run (push to remote repositories?) [y/n] +``` + +1. Wait for all repos to be checked out to complete. There will be some additional work done, e.g. along with branch and tag creation. In a live run, you may be prompted for Git credentials to push a branch. + +``` + Checking out repo: https://dist.apache.org/repos/dist/dev/metron + Checking out repo: dev + Checking out repo: https://dist.apache.org/repos/dist/release/metron + Checking out repo: release + Checking out git repo: https://git-wip-us.apache.org/repos/asf/metron.git + Cloning into '/Users/justinleet/tmp/metron-0.6.1/metron'... + remote: Counting objects: 46146, done. + remote: Compressing objects: 100% (15568/15568), done. + remote: Total 46146 (delta 21513), reused 43696 (delta 19489) + Receiving objects: 100% (46146/46146), 56.00 MiB | 1.04 MiB/s, done. + Resolving deltas: 100% (21513/21513), done. + Creating branch: Metron_0.6.1 + Using git rev: master + Already on 'master' + Your branch is up to date with 'origin/master'. + Switched to a new branch 'Metron_0.6.1' + This is a practice run. Not running + Creating tentative git tag <0.6.1-rc1>. Do not push this tag until RC is ready for community review. + Already on 'Metron_0.6.1' + Creating the RC tarball for tag apache-metron-0.6.1-rc1 + Creating the SHA
[19/51] [abbrv] metron git commit: METRON-1794 Include User Details When Escalating Alerts (nickwallen) closes apache/metron#1212
METRON-1794 Include User Details When Escalating Alerts (nickwallen) closes
apache/metron#1212
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c0fb2625
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c0fb2625
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c0fb2625
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: c0fb26258f9a78d375a50dd5d746404130dfe06b
Parents: 9c9e295
Author: nickwallen
Authored: Mon Oct 1 09:49:17 2018 -0400
Committer: nickallen
Committed: Mon Oct 1 09:49:17 2018 -0400
--
.../apache/metron/rest/MetronRestConstants.java | 3 +
.../rest/service/impl/AlertsUIServiceImpl.java | 53
.../service/impl/AlertsUIServiceImplTest.java | 63
.../src/test/resources/log4j.properties | 5 +-
4 files changed, 98 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/c0fb2625/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
index e3bf698..94e8e35 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -53,6 +53,9 @@ public class MetronRestConstants {
public static final String KAFKA_BROKER_URL_SPRING_PROPERTY =
"kafka.broker.url";
public static final String KAFKA_TOPICS_ESCALATION_PROPERTY =
"kafka.topics.escalation";
+ public static final String METRON_ESCALATION_USER_FIELD =
"metron_escalation_user";
+ public static final String METRON_ESCALATION_TIMESTAMP_FIELD =
"metron_escalation_timestamp";
+
public static final String KERBEROS_ENABLED_SPRING_PROPERTY =
"kerberos.enabled";
public static final String KERBEROS_PRINCIPLE_SPRING_PROPERTY =
"kerberos.principal";
public static final String KERBEROS_KEYTAB_SPRING_PROPERTY =
"kerberos.keytab";
http://git-wip-us.apache.org/repos/asf/metron/blob/c0fb2625/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
index 7d0a8f8..bf035e2 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/AlertsUIServiceImpl.java
@@ -19,26 +19,32 @@ package org.apache.metron.rest.service.impl;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.metron.common.system.Clock;
import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.hbase.client.UserSettingsClient;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.AlertsUIUserSettings;
-import org.apache.metron.hbase.client.UserSettingsClient;
import org.apache.metron.rest.security.SecurityUtils;
import org.apache.metron.rest.service.AlertsUIService;
import org.apache.metron.rest.service.KafkaService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static
org.apache.metron.rest.MetronRestConstants.KAFKA_TOPICS_ESCALATION_PROPERTY;
+import static
org.apache.metron.rest.MetronRestConstants.METRON_ESCALATION_TIMESTAMP_FIELD;
+import static
org.apache.metron.rest.MetronRestConstants.METRON_ESCALATION_USER_FIELD;
+
/**
* The default service layer implementation of {@link AlertsUIService}.
*
@@ -47,6 +53,7 @@ import org.springframework.stereotype.Service;
@Service
public class AlertsUIServiceImpl implements AlertsUIService {
+ static final Logger LOG =
[42/51] [abbrv] metron git commit: METRON-1820 Update to new Simple-Syslog-5424 version to support error handling (ottobackwards) closes apache/metron#1234
METRON-1820 Update to new Simple-Syslog-5424 version to support error handling
(ottobackwards) closes apache/metron#1234
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/08f3de0f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/08f3de0f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/08f3de0f
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 08f3de0fe31fefa828952cbe76456580a4697630
Parents: 6214150
Author: ottobackwards
Authored: Mon Oct 15 16:36:36 2018 -0400
Committer: otto
Committed: Mon Oct 15 16:36:36 2018 -0400
--
dependencies_with_url.csv | 2 +-
.../impl/SensorParserConfigServiceImpl.java | 65 +-
.../org/apache/metron/parsers/GrokParser.java | 18 +---
.../apache/metron/parsers/bolt/ParserBolt.java | 37 +++-
.../parsers/interfaces/MessageParser.java | 18
.../interfaces/MultilineMessageParser.java | 51 +++
.../metron/parsers/syslog/Syslog5424Parser.java | 44 --
.../apache/metron/parsers/GrokParserTest.java | 8 +-
.../metron/parsers/MultiLineGrokParserTest.java | 5 +-
.../metron/parsers/bolt/ParserBoltTest.java | 91 +++-
.../parsers/syslog/Syslog5424ParserTest.java| 55 +---
.../websphere/GrokWebSphereParserTest.java | 56
pom.xml | 2 +-
13 files changed, 335 insertions(+), 117 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index fb6c03c..2e1eedd 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -484,4 +484,4 @@ org.sonatype.sisu:sisu-inject-bean:jar:2.2.2:compile
org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
com.zaxxer:HikariCP:jar:2.7.8:compile,ASLv2,https://github.com/brettwooldridge/HikariCP
org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https://github.com/hibernate/hibernate-validator
-com.github.palindromicity:simple-syslog-5424:jar:0.0.8:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
+com.github.palindromicity:simple-syslog-5424:jar:0.0.9:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
http://git-wip-us.apache.org/repos/asf/metron/blob/08f3de0f/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index 85b84b8..4cd272e 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.fs.Path;
@@ -33,6 +34,8 @@ import
org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.parsers.interfaces.MultilineMessageParser;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.ParseMessageRequest;
@@ -138,13 +141,53 @@ public class SensorParserConfigServiceImpl implements
SensorParserConfigService
} else if (sensorParserConfig.getParserClassName() == null) {
throw new RestException("SensorParserConfig must have a
parserClassName");
} else {
- MessageParser parser;
+ MultilineMessageParser parser;
+ Object parserObject;
try {
-parser = (MessageParser)
Class.forName(sensorParserConfig.getParserClassName())
+parserObject = Class.forName(sensorParserConfig.getParserClassName())
.newInstance();
} catch (Exception e) {
throw new RestException(e.toString(), e.getCause());
}
+
+ if (!(parserObject instanceof MultilineMessageParser)) {
+parser = new MultilineMessageParser() {
+
+ @Override
+ @Suppre
[25/51] [abbrv] metron git commit: METRON-1771 Update REST endpoints to support eventually consistent UI updates (merrimanr) closes apache/metron#1190
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
--
diff --git
a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
index 4d48075..096baf1 100644
---
a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
+++
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
@@ -24,16 +24,17 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
@@ -80,15 +81,16 @@ public abstract class AbstractLuceneMetaAlertUpdateDao
implements MetaAlertUpdat
* @param retrieveLatestDao DAO to retrieve the item to be patched
* @param request The patch request.
* @param timestamp Optionally a timestamp to set. If not specified then
current time is used.
+ * @return The patched document
* @throws OriginalNotFoundException If no original document is found to
patch.
* @throws IOException If an error occurs performing the patch.
*/
@Override
- public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
+ public Document patch(RetrieveLatestDao retrieveLatestDao, PatchRequest
request,
Optional timestamp)
throws OriginalNotFoundException, IOException {
if (isPatchAllowed(request)) {
- updateDao.patch(retrieveLatestDao, request, timestamp);
+ return updateDao.patch(retrieveLatestDao, request, timestamp);
} else {
throw new IllegalArgumentException(
"Meta alert patches are not allowed for /alert or /status paths. "
@@ -97,7 +99,7 @@ public abstract class AbstractLuceneMetaAlertUpdateDao
implements MetaAlertUpdat
}
@Override
- public void batchUpdate(Map> updates) {
+ public Map> batchUpdate(Map> updates) {
throw new UnsupportedOperationException("Meta alerts do not allow for bulk
updates");
}
@@ -170,21 +172,65 @@ public abstract class AbstractLuceneMetaAlertUpdateDao
implements MetaAlertUpdat
return updates;
}
+ /**
+ * Adds alerts to a metaalert, based on a list of GetRequests provided for
retrieval.
+ * @param metaAlertGuid The GUID of the metaalert to be given new children.
+ * @param alertRequests GetRequests for the appropriate alerts to add.
+ * @return The updated metaalert with alerts added.
+ */
+ @Override
+ public Document addAlertsToMetaAlert(String metaAlertGuid, List
alertRequests)
+ throws IOException {
+Document metaAlert = retrieveLatestDao
+.getLatest(metaAlertGuid, MetaAlertConstants.METAALERT_TYPE);
+if (metaAlert == null) {
+ throw new IOException(String.format("Unable to add alerts to meta alert.
Meta alert with guid %s cannot be found.",
+ metaAlertGuid));
+}
+if (MetaAlertStatus.ACTIVE.getStatusString()
+
.equals(metaAlert.getDocument().get(MetaAlertConstants.STATUS_FIELD))) {
+ Iterable alerts =
retrieveLatestDao.getAllLatest(alertRequests);
+ Set missingAlerts = getMissingAlerts(alertRequests, alerts);
+ if (!missingAlerts.isEmpty()) {
+throw new IOException(String.format("Unable to add alerts to meta
alert. Alert with guid %s cannot be found.",
+missingAlerts.iterator().next()));
+ }
+ Map> updates =
buildAddAlertToMetaAlertUpdates(metaAlert, alerts);
+ update(updates);
+ return metaAlert;
+} else {
+ throw new IllegalStateException("Adding alerts to an INACTIVE meta alert
is not allowed");
+}
+ }
+
+ /**
+ * Removes alerts from a metaalert, based on a list of GetRequests provided
for retrieval.
+ * @param metaAlertGuid The GUID of the metaalert to remove children from.
+ * @param alertRequests A list of GetReqests that will provide the alerts to
remove
+ * @return The updated metaalert with alerts removed.
+ * @throws IllegalStateException If the metaalert is inactive.
+ */
[33/51] [abbrv] metron git commit: METRON-1811 Alert Search Fails When Sorting by Alert Status (merrimanr) closes apache/metron#1231
METRON-1811 Alert Search Fails When Sorting by Alert Status (merrimanr) closes
apache/metron#1231
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/747220f0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/747220f0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/747220f0
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 747220f000188557bfa860b48f6eb4bc49758d4a
Parents: 3467ffd
Author: merrimanr
Authored: Tue Oct 9 15:33:39 2018 -0500
Committer: rmerriman
Committed: Tue Oct 9 15:33:39 2018 -0500
--
.../CURRENT/package/files/bro_index.template| 3 +
.../CURRENT/package/files/snort_index.template | 3 +
.../CURRENT/package/files/yaf_index.template| 3 +
metron-platform/metron-elasticsearch/README.md | 20 +-
.../ElasticsearchSearchIntegrationTest.java | 211 ++-
.../indexing/dao/SearchIntegrationTest.java | 34 +--
.../src/main/config/schema/bro/schema.xml | 1 +
.../src/main/config/schema/snort/schema.xml | 1 +
.../src/main/config/schema/yaf/schema.xml | 1 +
.../integration/SolrSearchIntegrationTest.java | 8 +-
10 files changed, 112 insertions(+), 173 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
index 17ad4d2..48e76c7 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
@@ -133,6 +133,9 @@
"source:type": {
"type": "keyword"
},
+"alert_status": {
+ "type": "keyword"
+},
/*
* Widely-used Bro fields (potentially renamed during Metron ingest)
*/
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
index f7c6e59..139dbc9 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
@@ -123,6 +123,9 @@
"source:type": {
"type": "keyword"
},
+"alert_status": {
+ "type": "keyword"
+},
"ip_dst_addr": {
"type": "ip"
},
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
index f4093ba..4a3e5f9 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
@@ -123,6 +123,9 @@
"source:type": {
"type": "keyword"
},
+"alert_status": {
+ "type": "keyword"
+},
"ip_dst_addr": {
"type": "ip"
},
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-platform/metron-elasticsearch/README.md
--
diff --git a/metron-platform/metron-elasticsearch/README.md
b/metron-platform/metron-elasticsearch/README.md
index d889e27..177412e 100644
---
[44/51] [abbrv] metron git commit: METRON-1681 Decouple the ParserBolt from the Parse execution logic (merrimanr) closes apache/metron#1213
METRON-1681 Decouple the ParserBolt from the Parse execution logic (merrimanr)
closes apache/metron#1213
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/28542ad6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/28542ad6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/28542ad6
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 28542ad64cf63f17b728b4b1c0e995a8973767f7
Parents: 08f3de0
Author: merrimanr
Authored: Thu Oct 18 13:59:52 2018 -0500
Committer: rmerriman
Committed: Thu Oct 18 13:59:52 2018 -0500
--
.../impl/SensorParserConfigServiceImpl.java | 51 +-
.../parsers/DefaultParserRunnerResults.java | 71 ++
.../org/apache/metron/parsers/GrokParser.java | 3 +-
.../org/apache/metron/parsers/ParserRunner.java | 60 ++
.../apache/metron/parsers/ParserRunnerImpl.java | 322 +++
.../metron/parsers/ParserRunnerResults.java | 33 +
.../apache/metron/parsers/bolt/ParserBolt.java | 381 +++-
.../parsers/filters/BroMessageFilter.java | 2 +-
.../metron/parsers/filters/StellarFilter.java | 2 +-
.../parsers/interfaces/MessageFilter.java | 2 +-
.../parsers/interfaces/MessageParser.java | 27 +-
.../interfaces/MultilineMessageParser.java | 51 --
.../metron/parsers/syslog/Syslog5424Parser.java | 4 +-
.../parsers/topology/ParserComponent.java | 56 ++
.../parsers/topology/ParserComponents.java | 67 --
.../parsers/topology/ParserTopologyBuilder.java | 39 +-
.../org/apache/metron/filters/FiltersTest.java | 4 +-
.../metron/parsers/MessageParserTest.java | 108 ++-
.../metron/parsers/ParserRunnerImplTest.java| 390 +
.../metron/parsers/bolt/ParserBoltTest.java | 859 ++-
.../parsers/integration/ParserDriver.java | 60 +-
21 files changed, 1481 insertions(+), deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index 4cd272e..d0e4b3d 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -20,12 +20,10 @@ package org.apache.metron.rest.service.impl;
import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
import com.fasterxml.jackson.databind.ObjectMapper;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.fs.Path;
import org.apache.metron.common.configuration.ConfigurationType;
@@ -35,18 +33,14 @@ import
org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.interfaces.MessageParserResult;
-import org.apache.metron.parsers.interfaces.MultilineMessageParser;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.ParseMessageRequest;
import org.apache.metron.rest.service.GrokService;
import org.apache.metron.rest.service.SensorParserConfigService;
import org.apache.metron.rest.util.ParserIndex;
-import org.apache.metron.common.zookeeper.ZKConfigurationsCache;
import org.apache.zookeeper.KeeperException;
import org.json.simple.JSONObject;
-import org.reflections.Reflections;
-import org.reflections.util.ConfigurationBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -141,53 +135,13 @@ public class SensorParserConfigServiceImpl implements
SensorParserConfigService
} else if (sensorParserConfig.getParserClassName() == null) {
throw new RestException("SensorParserConfig must have a
parserClassName");
} else {
- MultilineMessageParser parser;
- Object parserObject;
+ MessageParser parser;
try {
-parserObject = Class.forName(sensorParserConfig.getParserClassName())
+parser = (MessageParser)
Class.forName(sensorParserConfig.getParserClassName())
.newInstance();
} catch (Exception e) {
throw new
[27/51] [abbrv] metron git commit: METRON-1695 Expose pcap properties through Ambari (anandsubbu) closes apache/metron#1207
METRON-1695 Expose pcap properties through Ambari (anandsubbu) closes
apache/metron#1207
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/feb9153a
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/feb9153a
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/feb9153a
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: feb9153a610e7cc384055c5d07b5c1dcf8b81923
Parents: de53306
Author: anandsubbu
Authored: Fri Oct 5 12:09:55 2018 +0530
Committer: anandsubbu
Committed: Fri Oct 5 12:09:55 2018 +0530
--
.../roles/ambari_config/vars/single_node_vm.yml | 2 +-
metron-deployment/packaging/ambari/.gitignore | 1 +
.../packaging/ambari/metron-mpack/pom.xml | 7 +
.../CURRENT/configuration/metron-pcap-env.xml | 188 +++
.../CURRENT/configuration/metron-rest-env.xml | 47 -
.../common-services/METRON/CURRENT/metainfo.xml | 3 +-
.../package/scripts/params/params_linux.py | 33 +++-
.../package/scripts/params/status_params.py | 2 +-
.../CURRENT/package/scripts/pcap_master.py | 7 +
.../METRON/CURRENT/themes/metron_theme.json | 174 +++--
metron-interface/metron-rest/README.md | 4 +-
metron-platform/metron-pcap-backend/README.md | 3 +
.../src/main/assembly/assembly.xml | 1 +
.../src/main/config/pcap.properties | 9 +-
.../src/main/config/pcap.properties.j2 | 40
15 files changed, 446 insertions(+), 75 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
--
diff --git
a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index 2ec956a..f67522c 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -105,7 +105,7 @@ configurations:
- metron-rest-env:
metron_spring_profiles_active: "dev"
- metron-parsers-env:
- parsers: "bro,snort"
+ parsers: "\"bro,snort,yaf\""
- elastic-site:
index_number_of_shards: 1
index_number_of_replicas: 0
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/.gitignore
--
diff --git a/metron-deployment/packaging/ambari/.gitignore
b/metron-deployment/packaging/ambari/.gitignore
index d2cc43c..242a4da 100644
--- a/metron-deployment/packaging/ambari/.gitignore
+++ b/metron-deployment/packaging/ambari/.gitignore
@@ -6,3 +6,4 @@ hdfs.properties.j2
enrichment.properties.j2
enrichment-splitjoin.properties.j2
enrichment-unified.properties.j2
+pcap.properties.j2
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
--
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index 82386c1..2975963 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -136,6 +136,13 @@
false
+
+
${basedir}/../../../../metron-platform/metron-pcap-backend/src/main/config
+
+pcap.properties.j2
+
+false
+
http://git-wip-us.apache.org/repos/asf/metron/blob/feb9153a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
new file mode 100644
index 000..31f9e8f
--- /dev/null
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-pcap-env.xml
@@ -0,0 +1,188 @@
+
+
+
+
+
+pcap_topology_worker_childopts
+PCAP Topology JVM Options
+
[26/51] [abbrv] metron git commit: METRON-1771 Update REST endpoints to support eventually consistent UI updates (merrimanr) closes apache/metron#1190
METRON-1771 Update REST endpoints to support eventually consistent UI updates (merrimanr) closes apache/metron#1190 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/de533063 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/de533063 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/de533063 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: de533063c26c8a2462dbfff33228a949c4a96c97 Parents: 7e222fa Author: merrimanr Authored: Thu Oct 4 17:16:18 2018 -0500 Committer: rmerriman Committed: Thu Oct 4 17:16:18 2018 -0500 -- metron-interface/metron-rest/README.md | 16 ++ .../rest/controller/MetaAlertController.java| 18 +- .../rest/controller/UpdateController.java | 31 ++- .../metron/rest/service/MetaAlertService.java | 12 +- .../metron/rest/service/UpdateService.java | 9 +- .../rest/service/impl/MetaAlertServiceImpl.java | 18 +- .../rest/service/impl/UpdateServiceImpl.java| 18 +- .../MetaAlertControllerIntegrationTest.java | 152 +++--- .../elasticsearch/dao/ElasticsearchDao.java | 32 +-- .../dao/ElasticsearchMetaAlertDao.java | 37 ++-- .../dao/ElasticsearchMetaAlertUpdateDao.java| 55 ++--- .../dao/ElasticsearchUpdateDao.java | 42 ++-- .../dao/ElasticsearchMetaAlertDaoTest.java | 27 ++- .../dao/ElasticsearchUpdateDaoTest.java | 52 + .../apache/metron/indexing/dao/HBaseDao.java| 31 +-- .../metron/indexing/dao/MultiIndexDao.java | 138 - .../dao/metaalert/MetaAlertCreateResponse.java | 40 .../dao/metaalert/MetaAlertUpdateDao.java | 20 +- .../AbstractLuceneMetaAlertUpdateDao.java | 81 +++- .../metron/indexing/dao/update/UpdateDao.java | 25 ++- .../InMemoryMetaAlertRetrieveLatestDao.java | 49 + .../metron/indexing/dao/HBaseDaoTest.java | 41 .../apache/metron/indexing/dao/InMemoryDao.java | 18 +- .../indexing/dao/InMemoryMetaAlertDao.java | 199 +-- .../dao/InMemoryMetaAlertUpdateDao.java | 91 + .../metron/indexing/dao/MultiIndexDaoTest.java | 96 + .../metron/indexing/dao/UpdateDaoTest.java | 74 +++ .../indexing/dao/UpdateIntegrationTest.java | 138 +++-- .../dao/metaalert/MetaAlertIntegrationTest.java | 119 --- .../AbstractLuceneMetaAlertUpdateDaoTest.java | 56 -- .../integration/HBaseDaoIntegrationTest.java| 7 +- .../org/apache/metron/solr/dao/SolrDao.java | 28 +-- .../metron/solr/dao/SolrMetaAlertDao.java | 49 +++-- .../metron/solr/dao/SolrMetaAlertUpdateDao.java | 38 ++-- .../apache/metron/solr/dao/SolrUpdateDao.java | 36 ++-- .../metron/solr/dao/SolrMetaAlertDaoTest.java | 21 +- .../metron/solr/dao/SolrUpdateDaoTest.java | 19 +- 37 files changed, 1206 insertions(+), 727 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/README.md -- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 2c216d1..2ce9522 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -336,6 +336,8 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `GET /api/v1/storm/supervisors`](#get-apiv1stormsupervisors)| | [ `PATCH /api/v1/update/patch`](#patch-apiv1updatepatch)| | [ `PUT /api/v1/update/replace`](#put-apiv1updatereplace)| +| [ `POST /api/v1/update/add/comment`](#put-apiv1updateaddcomment)| +| [ `POST /api/v1/update/remove/comment`](#put-apiv1updateremovecomment)| | [ `GET /api/v1/user`](#get-apiv1user)| ### `POST /api/v1/alerts/ui/escalate` @@ -963,6 +965,20 @@ Request and Response objects are JSON formatted. The JSON schemas are available ``` * Returns: * 200 - Current user + +### `POST /api/v1/update/add/comment` + * Description: Add a comment to an alert + * Input: +* request - Comment add request + * Returns: +* 200 - Returns the complete alert document with comments added. + +### `POST /api/v1/update/remove/comment` + * Description: Remove a comment from an alert + * Input: +* request - Comment remove request + * Returns: +* 200 - Returns the complete alert document with comments removed. ### `GET /api/v1/user` * Description: Retrieves the current user http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/MetaAlertController.java -- diff --git
[31/51] [abbrv] metron git commit: METRON-1806: Upgrade Maven Shade Plugin version (mmiklavc via mmiklavc) closes apache/metron#1224
METRON-1806: Upgrade Maven Shade Plugin version (mmiklavc via mmiklavc) closes apache/metron#1224 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e4823667 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e4823667 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e4823667 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: e482366726b07173fbc9ff9b084b851596e13005 Parents: 5bfc08c Author: mmiklavc Authored: Tue Oct 9 09:22:22 2018 -0600 Committer: Michael Miklavcic Committed: Tue Oct 9 09:22:22 2018 -0600 -- .gitignore | 1 + pom.xml| 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/e4823667/.gitignore -- diff --git a/.gitignore b/.gitignore index a0b9691..df3f030 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ target/ *.tmp *.bak *.class +*.pyc tmp/** tmp/**/* http://git-wip-us.apache.org/repos/asf/metron/blob/e4823667/pom.xml -- diff --git a/pom.xml b/pom.xml index a98de72..1e6adb0 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ 6.6.2 1.10.19 1.7.0 -2.4.3 +3.2.0 2.7.4 2.0.14 3.0.2
[21/51] [abbrv] metron git commit: METRON-1798 Add mpack support for parser aggregation (anandsubbu) closes apache/metron#1215
METRON-1798 Add mpack support for parser aggregation (anandsubbu) closes
apache/metron#1215
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ebdaf5f9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ebdaf5f9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ebdaf5f9
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: ebdaf5f905829d8ff59814bb3555666217f85abb
Parents: ff1f9cf
Author: anandsubbu
Authored: Wed Oct 3 18:57:48 2018 +0530
Committer: anandsubbu
Committed: Wed Oct 3 18:57:48 2018 +0530
--
.../configuration/metron-parsers-env.xml| 2 +-
.../CURRENT/package/scripts/parser_commands.py | 49 ++--
.../metron-parsers/ParserChaining.md| 18 +++
3 files changed, 63 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/ebdaf5f9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
index a9a498b..03a2594 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-parsers-env.xml
@@ -21,7 +21,7 @@
parsers
bro,snort,yaf
-Metron parsers to deploy
+Metron parsers to deploy. You can also specify an
aggregated parser list by grouping them with double quotes. For example:
"parserA,parserB",parserC,parserD
Metron Parsers
http://git-wip-us.apache.org/repos/asf/metron/blob/ebdaf5f9/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
index 274306a..18780d9 100755
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
@@ -20,6 +20,7 @@ limitations under the License.
import os
import re
+import shlex
import subprocess
import time
@@ -49,7 +50,45 @@ class ParserCommands:
# get list of parsers
def __get_parsers(self, params):
-return params.parsers.replace(' ', '').split(',')
+"""
+Combines the list of parser topics and sends a unique list to be used
for
+Kafka topic creation and the like.
+:param params:
+:return: List containing the names of unique parsers
+"""
+parserBatches = list(self.__get_aggr_parsers(params))
+parsers = ','.join(s.translate(None, '"') for s in parserBatches)
+# Get only the unique list of parser names
+parsers = list(set(parsers.split(',')))
+return parsers
+
+def __get_aggr_parsers(self, params):
+"""
+Fetches the list of aggregated (and regular) parsers and returns a
list.
+If the input list of parsers were "bro,snort,yaf", "bro,snort" and
yaf, for example,
+then this method will return ["bro,snort,yaf", "bro,snort", "yaf"]
+:param params:
+:return: List containing the names of parsers
+"""
+parserList = []
+parsers = shlex.shlex(params.parsers)
+for name in parsers:
+parserList.append(name.strip(','))
+return [s.translate(None, "'[]") for s in filter(None, parserList)]
+
+def get_parser_aggr_topology_names(self, params):
+"""
+Returns the names of regular and aggregated topologies as they would
run in storm
+An aggregated topology has the naming convention of 'parserA__parserB'.
+For example, a list of parsers like ["bro,snort", yaf] will be
returned as ["bro__snort", "yaf"]
+:param params:
+:return: List containing the names of parser topologies
+"""
+topologyName = []
+for parser in self.__get_aggr_parsers(params):
+
[45/51] [abbrv] metron git commit: METRON-1816 Date format Stellar function (merrimanr) closes apache/metron#1233
METRON-1816 Date format Stellar function (merrimanr) closes apache/metron#1233
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bf32fef6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bf32fef6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bf32fef6
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: bf32fef6a4a02c5b21ec1f6d97fbd83de78a996e
Parents: 28542ad
Author: merrimanr
Authored: Thu Oct 18 17:00:44 2018 -0500
Committer: rmerriman
Committed: Thu Oct 18 17:00:44 2018 -0500
--
metron-stellar/stellar-common/README.md | 9 +
.../stellar/dsl/functions/DateFunctions.java| 42
.../dsl/functions/DateFunctionsTest.java| 38 ++
3 files changed, 89 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/bf32fef6/metron-stellar/stellar-common/README.md
--
diff --git a/metron-stellar/stellar-common/README.md
b/metron-stellar/stellar-common/README.md
index 468f358..0f3bb6a 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -169,6 +169,7 @@ Where:
| [ `CHOP`](#chop)
|
| [ `CHOMP`](#chomp)
|
| [ `COUNT_MATCHES`](#count_matches)
|
+| [ `DATE_FORMAT`](#date_format)
| [ `DAY_OF_MONTH`](#day_of_month)
|
| [ `DAY_OF_WEEK`](#day_of_week)
|
| [ `DAY_OF_YEAR`](#day_of_year)
|
@@ -379,6 +380,14 @@ Where:
* substring/character - the substring or character to count, may be null.
* Returns: the number of non-overlapping occurrences, 0 if either
CharSequence is null.
+### `DATE_FORMAT`
+ * Description: Takes an epoch timestamp and converts it to a date format.
+ * Input:
+* format - DateTime format as a String.
+* timestampField - Optional epoch time in Long format. Defaults to now.
+* timezone - Optional timezone in String format.
+ * Returns: Formatted date.
+
### `DAY_OF_MONTH`
* Description: The numbered day within the month. The first day within the
month has a value of 1.
* Input:
http://git-wip-us.apache.org/repos/asf/metron/blob/bf32fef6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
index 212d6e9..17f5f8d 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
@@ -28,6 +28,7 @@ import org.apache.metron.stellar.common.utils.ConversionUtils;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
+import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.TimeZone;
@@ -109,6 +110,13 @@ public class DateFunctions {
return sdf.parse(date).getTime();
}
+ public static String getDateFormat(String format, Optional epochTime,
Optional timezone) {
+Long time = epochTime.orElseGet(System::currentTimeMillis);
+TimezonedFormat fmt = timezone.map(s -> new TimezonedFormat(format,
s)).orElseGet(() -> new TimezonedFormat(format));
+SimpleDateFormat sdf = formatCache.get(fmt).get();
+return sdf.format(new Date(time));
+ }
+
/**
* Stellar Function: TO_EPOCH_TIMESTAMP
@@ -144,6 +152,40 @@ public class DateFunctions {
}
}
+ @Stellar( name="DATE_FORMAT",
+ description = "Takes an epoch timestamp and converts it to a date
format.",
+ params = {"format - DateTime format as a String."
+ , "timestampField - Optional epoch time in Long format.
Defaults to now."
+ , "timezone - Optional timezone in String format."},
+ returns="Formatted date."
+ )
+ public static class DateFormat extends BaseStellarFunction {
+
+public Object apply(List objects) {
+ int size = objects.size();
+ Optional formatObj = Optional.ofNullable(objects.get(0));
+ Optional epochObj = Optional.empty();
+ Optional tzObj = Optional.empty();
+
[47/51] [abbrv] metron git commit: METRON-1829 Large Error Message Causes Slow Search Performance (merrimanr) closes apache/metron#1239
METRON-1829 Large Error Message Causes Slow Search Performance (merrimanr)
closes apache/metron#1239
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d44a3925
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d44a3925
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d44a3925
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: d44a392567e35022bfb35a787b55aff3447ac60e
Parents: 2531c3e
Author: merrimanr
Authored: Mon Oct 22 08:43:10 2018 -0500
Committer: rmerriman
Committed: Mon Oct 22 08:43:10 2018 -0500
--
.../bolt/BulkMessageWriterBoltTest.java | 2 +-
metron-platform/metron-writer/pom.xml | 6 ++
.../metron/writer/BulkWriterComponent.java | 33 ++-
.../writer/bolt/BulkMessageWriterBolt.java | 2 +-
.../metron/writer/BulkWriterComponentTest.java | 61 +++-
5 files changed, 76 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
--
diff --git
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
index 588fc58..083628c 100644
---
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
+++
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
@@ -207,7 +207,7 @@ public class BulkMessageWriterBoltTest extends
BaseEnrichmentBoltTest {
}
UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.ERROR);
verify(outputCollector, times(5)).ack(tuple);
-verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM),
any(Values.class));
+verify(outputCollector, times(5)).emit(eq(Constants.ERROR_STREAM),
any(Values.class));
verify(outputCollector, times(1)).reportError(any(Throwable.class));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-writer/pom.xml
--
diff --git a/metron-platform/metron-writer/pom.xml
b/metron-platform/metron-writer/pom.xml
index e845516..6d08093 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/pom.xml
@@ -207,6 +207,12 @@
metron-common
${project.parent.version}
+
+org.apache.metron
+metron-test-utilities
+${project.parent.version}
+test
+
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
--
diff --git
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
index 7678584..68585c5 100644
---
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
+++
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
@@ -40,6 +40,7 @@ import org.apache.metron.common.writer.BulkMessageWriter;
import org.apache.metron.common.writer.BulkWriterResponse;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -118,12 +119,18 @@ public class BulkWriterComponent {
public void error(String sensorType, Throwable e, Iterable tuples,
MessageGetStrategy messageGetStrategy) {
LOG.error(format("Failing %d tuple(s); sensorType=%s",
Iterables.size(tuples), sensorType), e);
-MetronError error = new MetronError()
-.withSensorType(Collections.singleton(sensorType))
-.withErrorType(Constants.ErrorType.INDEXING_ERROR)
-.withThrowable(e);
-tuples.forEach(t -> error.addRawMessage(messageGetStrategy.get(t)));
-handleError(tuples, error);
+tuples.forEach(t -> {
+ MetronError error = new MetronError()
+ .withSensorType(Collections.singleton(sensorType))
+ .withErrorType(Constants.ErrorType.INDEXING_ERROR)
+ .withThrowable(e)
+ .addRawMessage(messageGetStrategy.get(t));
+ collector.emit(Constants.ERROR_STREAM, new
Values(error.getJSONObject()));
+ collector.ack(t);
+});
+//
[20/51] [abbrv] metron git commit: METRON-1750 Create Parser for Syslog RFC 5424 Messages (ottobackwards) closes apache/metron#1175
METRON-1750 Create Parser for Syslog RFC 5424 Messages (ottobackwards) closes
apache/metron#1175
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ff1f9cf5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ff1f9cf5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ff1f9cf5
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: ff1f9cf52b31ae866eb3de1cf8993af97af1790f
Parents: c0fb262
Author: ottobackwards
Authored: Mon Oct 1 10:23:38 2018 -0400
Committer: otto
Committed: Mon Oct 1 10:23:38 2018 -0400
--
dependencies_with_url.csv | 2 +-
.../docker/rpm-docker/SPECS/metron.spec | 3 +
.../data/syslog5424/parsed/Syslog5424Parsed | 3 +
.../sample/data/syslog5424/raw/Syslog5424Output | 3 +
metron-platform/metron-parsers/README.md| 5 +-
metron-platform/metron-parsers/pom.xml | 5 +
.../config/zookeeper/parsers/syslog5424.json| 7 +
.../metron/parsers/syslog/Syslog5424Parser.java | 102 +
.../Syslog5424ParserIntegrationTest.java| 37 +
.../parsers/syslog/Syslog5424ParserTest.java| 146 +++
pom.xml | 1 +
11 files changed, 311 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index ffd0fbc..53977f3 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -488,4 +488,4 @@
com.google.code.gson:gson:jar:2.8.2:compile,ASLv2,https://github.com/google/gson
org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
com.zaxxer:HikariCP:jar:2.7.8:compile,ASLv2,https://github.com/brettwooldridge/HikariCP
org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https://github.com/hibernate/hibernate-validator
-
+com.github.palindromicity:simple-syslog-5424:jar:0.0.8:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 4bcef33..ed22a28 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -158,6 +158,7 @@ This package installs the Metron Parser files
%{metron_home}/config/zookeeper/parsers/jsonMapWrappedQuery.json
%{metron_home}/config/zookeeper/parsers/snort.json
%{metron_home}/config/zookeeper/parsers/squid.json
+%{metron_home}/config/zookeeper/parsers/syslog5424.json
%{metron_home}/config/zookeeper/parsers/websphere.json
%{metron_home}/config/zookeeper/parsers/yaf.json
%{metron_home}/config/zookeeper/parsers/asa.json
@@ -590,6 +591,8 @@ chkconfig --del metron-alerts-ui
%changelog
* Thu Aug 30 2018 Apache Metron - 0.6.1
- Update compiled css file name for Alerts UI
+* Fri Aug 24 2018 Apache Metron - 0.6.1
+- Add syslog5424 parser
* Tue Aug 21 2018 Apache Metron - 0.6.1
- Add Profiler for REPL
* Tue Aug 14 2018 Apache Metron - 0.5.1
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
--
diff --git
a/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
new file mode 100644
index 000..e330204
--- /dev/null
+++
b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
@@ -0,0 +1,3 @@
+{"syslog.header.appName":"d0602076-b14a-4c55-852a-981e7afeed38","syslog.header.version":"1","syslog.header.hostName":"loggregator","original_string":"<14>1
2014-06-20T09:14:07+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed38 DEA
- - Removing
instance","syslog.header.facility":"1","syslog.header.msgId":"-","syslog.header.timestamp":"2014-06-20T09:14:07+00:00","syslog.message":"Removing
instance","syslog.header.pri":"14","syslog.header.procId":"DEA","syslog.header.severity":"6","timestamp":"2014-06-20T09:14:07+00:00","guid":"
[28/51] [abbrv] metron git commit: METRON-1808 Add Ansible created pyc to gitignore (justinleet) closes apache/metron#1228
METRON-1808 Add Ansible created pyc to gitignore (justinleet) closes apache/metron#1228 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/891bf495 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/891bf495 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/891bf495 Branch: refs/heads/feature/METRON-1090-stellar-assignment Commit: 891bf4952ab8eceabcb44659e7498d62005b6aa2 Parents: feb9153 Author: justinleet Authored: Fri Oct 5 16:14:20 2018 -0400 Committer: leet Committed: Fri Oct 5 16:14:20 2018 -0400 -- metron-deployment/.gitignore | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/metron/blob/891bf495/metron-deployment/.gitignore -- diff --git a/metron-deployment/.gitignore b/metron-deployment/.gitignore index 6038b6b..f3d4a2a 100644 --- a/metron-deployment/.gitignore +++ b/metron-deployment/.gitignore @@ -1,3 +1,4 @@ keys/ .vagrant *.retry +*.pyc
[18/51] [abbrv] metron git commit: METRON-1782 Add Kafka Partition and Offset to Profiler Debug Logs (nickwallen) closes apache/metron#1202
METRON-1782 Add Kafka Partition and Offset to Profiler Debug Logs (nickwallen)
closes apache/metron#1202
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9c9e2954
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9c9e2954
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9c9e2954
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 9c9e29547837e648723920329c3b7fea6211f0db
Parents: 1723a0e
Author: nickwallen
Authored: Mon Oct 1 09:07:28 2018 -0400
Committer: nickallen
Committed: Mon Oct 1 09:07:28 2018 -0400
--
.../src/main/flux/profiler/remote.yaml | 11 +--
.../metron/profiler/storm/ProfileSplitterBolt.java | 13 -
.../metron/profiler/storm/ProfileSplitterBoltTest.java | 3 ++-
.../storm/kafka/flux/SimpleStormKafkaBuilder.java | 4 +++-
4 files changed, 26 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/9c9e2954/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
--
diff --git
a/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
b/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
index da71b27..e16a782 100644
--- a/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
+++ b/metron-analytics/metron-profiler-storm/src/main/flux/profiler/remote.yaml
@@ -77,8 +77,15 @@ components:
className: "java.util.ArrayList"
configMethods:
- name: "add"
-args:
-- "value"
+args: ["value"]
+- name: "add"
+args: ["topic"]
+- name: "add"
+args: ["partition"]
+- name: "add"
+args: ["offset"]
+- name: "add"
+args: ["timestamp"]
- id: "kafkaConfig"
className: "org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder"
http://git-wip-us.apache.org/repos/asf/metron/blob/9c9e2954/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
--
diff --git
a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
index 81179b6..ef58ad9 100644
---
a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
+++
b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java
@@ -43,6 +43,12 @@ import java.lang.invoke.MethodHandles;
import java.util.List;
import java.util.Map;
+import static
org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.OFFSET;
+import static
org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.PARTITION;
+import static
org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.TIMESTAMP;
+import static
org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.TOPIC;
+import static
org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder.FieldsConfiguration.VALUE;
+
/**
* The Storm bolt responsible for filtering incoming messages and directing
* each to the downstream bolts responsible for building a Profile.
@@ -132,6 +138,11 @@ public class ProfileSplitterBolt extends
ConfiguredProfilerBolt {
@Override
public void execute(Tuple input) {
try {
+ LOG.debug("Received message; topic={}, partition={}, offset={},
kafkaTimestamp={}",
+ input.contains(TOPIC.getFieldName()) ?
input.getStringByField(TOPIC.getFieldName()): "unknown",
+ input.contains(PARTITION.getFieldName()) ?
input.getIntegerByField(PARTITION.getFieldName()): "unknown",
+ input.contains(OFFSET.getFieldName()) ?
input.getLongByField(OFFSET.getFieldName()):"unknown",
+ input.contains(TIMESTAMP.getFieldName()) ?
input.getLongByField(TIMESTAMP.getFieldName()): "unknown");
doExecute(input);
} catch (Throwable t) {
@@ -146,7 +157,7 @@ public class ProfileSplitterBolt extends
ConfiguredProfilerBolt {
private void doExecute(Tuple input) throws ParseException,
UnsupportedEncodingException {
// retrieve the input message
-byte[] data = input.getBinary(0);
+byte[] data = input.getBinaryByField(VALUE.getFieldName());
if(data == null) {
LOG.debug("Received null message. Nothing to do.");
return;
[15/51] [abbrv] metron git commit: METRON-1784: Re-allow remote ssh and scp in Centos full dev (mmiklavc via mmiklavc) closes apache/metron#1204
METRON-1784: Re-allow remote ssh and scp in Centos full dev (mmiklavc via
mmiklavc) closes apache/metron#1204
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4f0b608d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4f0b608d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4f0b608d
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: 4f0b608d77ebed3025ba0accedec440b3270cb18
Parents: 08fe1cf
Author: mmiklavc
Authored: Wed Sep 26 16:34:13 2018 -0600
Committer: Michael Miklavcic
Committed: Wed Sep 26 16:34:13 2018 -0600
--
.../roles/enable-remote-ssh/defaults/main.yml | 18 +++
.../roles/enable-remote-ssh/tasks/main.yml | 33
.../development/centos6/ansible/playbook.yml| 5 +--
3 files changed, 54 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/4f0b608d/metron-deployment/ansible/roles/enable-remote-ssh/defaults/main.yml
--
diff --git
a/metron-deployment/ansible/roles/enable-remote-ssh/defaults/main.yml
b/metron-deployment/ansible/roles/enable-remote-ssh/defaults/main.yml
new file mode 100644
index 000..f176095
--- /dev/null
+++ b/metron-deployment/ansible/roles/enable-remote-ssh/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+sshd_config_file: /etc/ssh/sshd_config
http://git-wip-us.apache.org/repos/asf/metron/blob/4f0b608d/metron-deployment/ansible/roles/enable-remote-ssh/tasks/main.yml
--
diff --git a/metron-deployment/ansible/roles/enable-remote-ssh/tasks/main.yml
b/metron-deployment/ansible/roles/enable-remote-ssh/tasks/main.yml
new file mode 100644
index 000..be83aa2
--- /dev/null
+++ b/metron-deployment/ansible/roles/enable-remote-ssh/tasks/main.yml
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Comment password auth no
+ lineinfile:
+dest: "{{ sshd_config_file }}"
+regexp: "^PasswordAuthentication no"
+line: "#PasswordAuthentication no"
+
+- name: Uncomment password auth yes
+ lineinfile:
+dest: "{{ sshd_config_file }}"
+regexp: "^#PasswordAuthentication yes"
+line: "PasswordAuthentication yes"
+
+- name: Restart service sshd, in all cases
+ service:
+name: sshd
+state: restarted
http://git-wip-us.apache.org/repos/asf/metron/blob/4f0b608d/metron-deployment/development/centos6/ansible/playbook.yml
--
diff --git a/metron-deployment/development/centos6/ansible/playbook.yml
b/metron-deployment/development/centos6/ansible/playbook.yml
index fa14677..562fb1d 100644
--- a/metron-deployment/development/centos6/ansible/playbook.yml
+++ b/metron-deployment/development/centos6/ansible/playbook.yml
@@ -17,7 +17,8 @@
---
- hosts: all
roles:
-- role: libselinux-python
-- role: enable-swap
+- { role: libselinux-python, tags: libselinux-python }
+- { role: enable-swap, tags: enable-swap }
+- { role: enable-remote-ssh, tags: enable-remote-ssh }
- include: ../../../ansible/playbooks/metron_full_install.yml
