[metron] branch master updated: METRON-2324 verify_licenses.sh and list_dependencies.sh don't properly handle maven failures (justinleet) closes apache/metron#1568
This is an automated email from the ASF dual-hosted git repository.
leet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 83effe4 METRON-2324 verify_licenses.sh and list_dependencies.sh don't
properly handle maven failures (justinleet) closes apache/metron#1568
83effe4 is described below
commit 83effe49b5b996af16f68b67c6e998010a51513f
Author: justinleet
AuthorDate: Fri Dec 6 19:11:19 2019 -0500
METRON-2324 verify_licenses.sh and list_dependencies.sh don't properly
handle maven failures (justinleet) closes apache/metron#1568
---
.travis.yml| 4 ++--
dependencies_with_url.csv | 1 +
dev-utilities/build-utils/list_dependencies.sh | 14 -
dev-utilities/build-utils/verify_licenses.sh | 23 +-
.../scripts/prepend_license_header.sh | 4 +++-
.../scripts/prepend_license_header.sh | 4 +++-
6 files changed, 44 insertions(+), 6 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 02fdd28..a0d8d13 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -36,12 +36,12 @@ matrix:
include:
- name: Unit Tests
script:
-- time mvn install -T 2C -q -DskipTests=true -Dmaven.javadoc.skip=true
-B -V -pl '!:metron-config,!:metron-alerts'
+- time mvn install -T 2C -q -DskipTests=true -Dmaven.javadoc.skip=true
-Dskip.npm -B -V
- time mvn surefire:test@unit-tests -T 2C
- name: Integration Tests
script:
-- time mvn install -T 2C -q -DskipTests=true -Dmaven.javadoc.skip=true
-B -V -pl '!:metron-config,!:metron-alerts'
+- time mvn install -T 2C -q -DskipTests=true -Dmaven.javadoc.skip=true
-Dskip.npm -B -V
- time mvn surefire:test@integration-tests
- name: Alerts UI Tests
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index c4fa12d..a716e48 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -50,6 +50,7 @@
javax.xml.bind:jaxb-api:jar:2.2.2:compile,CDDL,https://jaxb.dev.java.net/
javax.xml.bind:jaxb-api:jar:2.3.0:compile,CDDL,https://jaxb.dev.java.net/
javax.xml.stream:stax-api:jar:1.0-2:compile,COMMON DEVELOPMENT AND
DISTRIBUTION LICENSE (CDDL) Version
1.0,https://docs.oracle.com/javase/7/docs/api/javax/xml/stream/package-summary.html
jline:jline:jar:0.9.94:compile,BSD,http://jline.sourceforge.net
+junit:junit:jar:4.12:compile,Eclipse Public License 1.0,http://junit.org
net.razorvine:pyrolite:jar:4.13:compile,MIT,https://github.com/irmen/Pyrolite
net.sf.jopt-simple:jopt-simple:jar:3.2:compile,The MIT
License,http://jopt-simple.sourceforge.net
net.sf.jopt-simple:jopt-simple:jar:4.6:compile,The MIT
License,http://jopt-simple.sourceforge.net
diff --git a/dev-utilities/build-utils/list_dependencies.sh
b/dev-utilities/build-utils/list_dependencies.sh
index 4e81b7b..d5f8e4a 100755
--- a/dev-utilities/build-utils/list_dependencies.sh
+++ b/dev-utilities/build-utils/list_dependencies.sh
@@ -16,4 +16,16 @@
# limitations under the License.
#
-{ mvn dependency:list || { echo "ERROR: Failed to run mvn dependency:list" ;
exit 1 ; } ; mvn dependency:list -PHDP-2.5.0.0 || { echo "ERROR: Failed to run
mvn dependency:list -PHDP-2.5.0.0" ; exit 1 ; } ; } | grep "^\[INFO\] " | awk
'{print $2}' | grep -v "org.apache" | grep -v "test" | grep -v "provided" |
grep -v "runtime" | grep -v ":system" | sort | uniq
+DEPS=$(mvn dependency:list)
+rc=$?
+if [[ $rc != 0 ]]; then
+ echo "ERROR: Failed to run mvn dependency:list"
+ DEPS=$(mvn dependency:list -PHDP-2.5.0.0)
+ rc=$?
+ if [[ $rc != 0 ]]; then
+echo "ERROR: Failed to run mvn dependency:list -PHDP-2.5.0.0"
+exit $rc
+ fi
+fi
+
+echo "$DEPS" | grep "^\[INFO\] " | awk '{print $2}' | grep -v "org.apache" |
grep -v "test" | grep -v "provided" | grep -v "runtime" | grep -v ":system" |
sort | uniq
diff --git a/dev-utilities/build-utils/verify_licenses.sh
b/dev-utilities/build-utils/verify_licenses.sh
index ffd6e48..13b4fb2 100755
--- a/dev-utilities/build-utils/verify_licenses.sh
+++ b/dev-utilities/build-utils/verify_licenses.sh
@@ -16,4 +16,25 @@
# limitations under the License.
#
-dev-utilities/build-utils/list_dependencies.sh | python
dev-utilities/build-utils/verify_license.py ./dependencies_with_url.csv
+# Need to build before we can properly list dependencies
+echo "Building Metron"
+mvn install -T 2C -q -DskipTests=true \
+ -Dmaven.javadoc.skip=true \
+ -Dskip.npm \
+ -B -V
+
+echo "Determining dependencies"
+DEPS=$(dev-utilities/build-utils/list_dependencies.sh)
+rc=$?
+if [[ $rc != 0 ]]; then
+ echo "Failed to determine dependencies"
+ exit $rc
+fi
[metron] branch master updated: METRON-2311 Remove JUnit from all our uber jars (justinleet) closes apache/metron#1561
This is an automated email from the ASF dual-hosted git repository. leet pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron.git The following commit(s) were added to refs/heads/master by this push: new 97d7e46 METRON-2311 Remove JUnit from all our uber jars (justinleet) closes apache/metron#1561 97d7e46 is described below commit 97d7e46f53376303e18540075b0aa53198b0a1cb Author: justinleet AuthorDate: Tue Nov 19 20:46:38 2019 -0500 METRON-2311 Remove JUnit from all our uber jars (justinleet) closes apache/metron#1561 --- dependencies_with_url.csv | 2 - metron-analytics/metron-maas-service/pom.xml | 3 ++ metron-analytics/metron-profiler-client/pom.xml| 3 ++ metron-analytics/metron-profiler-repl/pom.xml | 7 +++ metron-analytics/metron-profiler-spark/pom.xml | 3 ++ metron-analytics/metron-profiler-storm/pom.xml | 3 ++ metron-analytics/metron-statistics/pom.xml | 3 ++ metron-contrib/metron-performance/pom.xml | 3 ++ metron-interface/metron-rest/pom.xml | 16 +++ metron-platform/elasticsearch-shaded/pom.xml | 3 ++ metron-platform/metron-common/pom.xml | 9 metron-platform/metron-data-management/pom.xml | 4 +- .../metron-elasticsearch-storm/pom.xml | 3 ++ .../metron-enrichment-common/pom.xml | 6 +++ .../metron-enrichment-storm/pom.xml| 6 +++ metron-platform/metron-hbase-client/pom.xml| 6 +++ metron-platform/metron-hbase-server/pom.xml| 3 ++ .../metron-indexing/metron-indexing-common/pom.xml | 3 ++ metron-platform/metron-management/pom.xml | 3 ++ .../metron-parsing/metron-parsers-common/pom.xml | 6 +++ .../metron-parsing/metron-parsers/pom.xml | 6 +++ .../metron-parsing/metron-parsing-storm/pom.xml| 6 +++ metron-platform/metron-pcap-backend/pom.xml| 3 ++ metron-platform/metron-pcap/pom.xml| 6 +++ .../metron-solr/metron-solr-common/pom.xml | 3 ++ .../metron-solr/metron-solr-storm/pom.xml | 3 ++ .../metron-storm-kafka-override/pom.xml| 3 ++ metron-platform/metron-test-utilities/pom.xml | 6 +++ .../metron-writer/metron-writer-common/pom.xml | 3 ++ metron-stellar/stellar-common/pom.xml | 13 +- .../common/utils/StellarProcessorUtils.java| 53 -- 31 files changed, 172 insertions(+), 28 deletions(-) diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index 595cf0e..c4fa12d 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -50,8 +50,6 @@ javax.xml.bind:jaxb-api:jar:2.2.2:compile,CDDL,https://jaxb.dev.java.net/ javax.xml.bind:jaxb-api:jar:2.3.0:compile,CDDL,https://jaxb.dev.java.net/ javax.xml.stream:stax-api:jar:1.0-2:compile,COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0,https://docs.oracle.com/javase/7/docs/api/javax/xml/stream/package-summary.html jline:jline:jar:0.9.94:compile,BSD,http://jline.sourceforge.net -junit:junit:jar:4.12:compile,Eclipse Public License 1.0,http://junit.org -junit:junit:jar:4.4:compile,Common Public License Version 1.0,http://junit.org net.razorvine:pyrolite:jar:4.13:compile,MIT,https://github.com/irmen/Pyrolite net.sf.jopt-simple:jopt-simple:jar:3.2:compile,The MIT License,http://jopt-simple.sourceforge.net net.sf.jopt-simple:jopt-simple:jar:4.6:compile,The MIT License,http://jopt-simple.sourceforge.net diff --git a/metron-analytics/metron-maas-service/pom.xml b/metron-analytics/metron-maas-service/pom.xml index a1671b5..d0dde0b 100644 --- a/metron-analytics/metron-maas-service/pom.xml +++ b/metron-analytics/metron-maas-service/pom.xml @@ -240,6 +240,9 @@ *slf4j* + + junit + org.junit*:* diff --git a/metron-analytics/metron-profiler-client/pom.xml b/metron-analytics/metron-profiler-client/pom.xml index 876e1d6..9335742 100644 --- a/metron-analytics/metron-profiler-client/pom.xml +++ b/metron-analytics/metron-profiler-client/pom.xml @@ -155,6 +155,9 @@ *slf4j* + +junit +org.junit*:* diff --git a/metron-analytics/metron-profiler-repl/pom.xml b/metron-analytics/metron-profiler-repl/pom.xml index f3714b8..010d272 100644 --- a/metron-analytics/metron-profiler-repl/pom.xml +++ b/metron-analytics/metron-profiler-repl/pom.xml @@ -105,6 +105,13 @@ true uber true + + + +junit +org.junit
svn commit: r36350 - in /release/metron/metron-bro-plugin-kafka: ./ 0.3.0/
Author: leet Date: Tue Oct 15 21:50:05 2019 New Revision: 36350 Log: Adding artifacts for Metron-bro-plugin-kafka 0.3.0 Added: release/metron/metron-bro-plugin-kafka/ release/metron/metron-bro-plugin-kafka/0.3.0/ release/metron/metron-bro-plugin-kafka/0.3.0/CHANGES release/metron/metron-bro-plugin-kafka/0.3.0/COPYING release/metron/metron-bro-plugin-kafka/0.3.0/NOTICE release/metron/metron-bro-plugin-kafka/0.3.0/apache-metron-bro-plugin-kafka_0.3.0.tar.gz (with props) release/metron/metron-bro-plugin-kafka/0.3.0/apache-metron-bro-plugin-kafka_0.3.0.tar.gz.asc release/metron/metron-bro-plugin-kafka/0.3.0/apache-metron-bro-plugin-kafka_0.3.0.tar.gz.sha256 release/metron/metron-bro-plugin-kafka/0.3.0/apache-metron-bro-plugin-kafka_0.3.0.tar.gz.sha512 Added: release/metron/metron-bro-plugin-kafka/0.3.0/CHANGES == --- release/metron/metron-bro-plugin-kafka/0.3.0/CHANGES (added) +++ release/metron/metron-bro-plugin-kafka/0.3.0/CHANGES Tue Oct 15 21:50:05 2019 @@ -0,0 +1,25 @@ +METRON-2269 Cannot run Docker tests if src is not a git repo (ottobackwards) closes apache/metron-bro-plugin-kafka#37 +METRON-2069 Add btests for bro plugin topic_name selection (ottobackwards) closes apache/metron-bro-plugin-kafka#36 +METRON-2045 Pass a version argument to the bro plugin docker scripts (JonZeolla) closes apache/metron-bro-plugin-kafka#35 +METRON-2003 Bro plugin topic should fall back to the log writer's path (JonZeolla) closes apache/metron-bro-plugin-kafka#26 +METRON-2025 Bro Kafka Plugin Docker should yum clean (ottobackwards) closes apache/metron-bro-plugin-kafka#33 +METRON-2021 Add screen to bro docker image (ottobackwards) closes apache/metron-bro-plugin-kafka#32 +METRON-2013 The bro plugin docker scripts topic name should be configurable (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#27 +METRON-2020 Running run_end_to_end.sh with docker give warning if bash 4.0 (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#31 +METRON-1991 Bro plugin docker scripts should exit nonzero when bro and kafka counts differ (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#29 +METRON-2017 The Bro plugin docker data processing script incorrectly runs bro (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#30 +METRON-1990 Bro plugin docker should exit nonzero if it encounters issues (JonZeolla) closes apache/metron-bro-plugin-kafka#28 +METRON-2004 Bro plugin kafka docker_execute_shell.sh workdir should be unspecified (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#25 +METRON-2000 Fix bro plugin docker line counting for BRO_COUNT (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#24 +METRON-1992 Support sending a log to multiple topics (JonZeolla) closes apache/metron-bro-plugin-kafka#23 +METRON-1910 bro plugin segfaults on src/KafkaWriter.cc:72 (JonZeolla) closes apache/metron-bro-plugin-kafka#20 +METRON-1911 Create Docker based test environment for Bro Kafka Plugin (ottobackwards) closes apache/metron-bro-plugin-kafka#21 +METRON-1885 Remove version from bro plugin btest (JonZeolla) closes apache/metron-bro-plugin-kafka#19 +METRON-1827 Update librdkafka in metron-bro-plugin-kafka (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#13 +METRON-1866 Improve metron-bro-plugin-kafka documentation (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#17 +METRON-1304 Allow metron-bro-plugin-kafka to include or exclude logs (JonZeolla via nickwallen) closes apache/metron-bro-plugin-kafka#2 +METRON-1865 Fix metron-bro-plugin-kafka tests (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#16 +METRON-1828 Improve bro plugin contributing documentation (JonZeolla) closes apache/metron-bro-plugin-kafka#14 +METRON-1818 Remove config_files from bro-pkg.meta (JonZeolla) closes apache/metron-bro-plugin-kafka#11 +METRON-1800 Increment metron-bro-plugin-kafka version (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#10 +METRON-1773 Bro plugin docs should refer to Apache Metron project (nickwallen) closes apache/metron-bro-plugin-kafka#9 Added: release/metron/metron-bro-plugin-kafka/0.3.0/COPYING == --- release/metron/metron-bro-plugin-kafka/0.3.0/COPYING (added) +++ release/metron/metron-bro-plugin-kafka/0.3.0/COPYING Tue Oct 15 21:50:05 2019 @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 +http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution
[metron-bro-plugin-kafka] tag 0.3 created (now abbbc9b)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to tag 0.3 in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git. at abbbc9b (commit) No new revisions were added by this update.
[metron-bro-plugin-kafka] tag apache-metron-bro-plugin-kafka_0.3.0-release created (now abbbc9b)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to tag apache-metron-bro-plugin-kafka_0.3.0-release in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git. at abbbc9b (commit) No new revisions were added by this update.
[metron-bro-plugin-kafka] tag apache-metron-bro-plugin-kafka_0.3.0-rc3 created (now abbbc9b)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to tag apache-metron-bro-plugin-kafka_0.3.0-rc3 in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git. at abbbc9b (commit) No new revisions were added by this update.
svn commit: r36281 - /dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/
Author: leet Date: Thu Oct 10 18:27:50 2019 New Revision: 36281 Log: Adding artifacts for metron-bro-plugin-kafka 0.3.0-RC3 Added: dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/ dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/CHANGES dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/COPYING dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/NOTICE dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/apache-metron-bro-plugin-kafka_0.3.0-rc3.tar.gz (with props) dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/apache-metron-bro-plugin-kafka_0.3.0-rc3.tar.gz.asc dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/apache-metron-bro-plugin-kafka_0.3.0-rc3.tar.gz.sha256 dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/apache-metron-bro-plugin-kafka_0.3.0-rc3.tar.gz.sha512 Added: dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/CHANGES == --- dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/CHANGES (added) +++ dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/CHANGES Thu Oct 10 18:27:50 2019 @@ -0,0 +1,25 @@ +METRON-2269 Cannot run Docker tests if src is not a git repo (ottobackwards) closes apache/metron-bro-plugin-kafka#37 +METRON-2069 Add btests for bro plugin topic_name selection (ottobackwards) closes apache/metron-bro-plugin-kafka#36 +METRON-2045 Pass a version argument to the bro plugin docker scripts (JonZeolla) closes apache/metron-bro-plugin-kafka#35 +METRON-2003 Bro plugin topic should fall back to the log writer's path (JonZeolla) closes apache/metron-bro-plugin-kafka#26 +METRON-2025 Bro Kafka Plugin Docker should yum clean (ottobackwards) closes apache/metron-bro-plugin-kafka#33 +METRON-2021 Add screen to bro docker image (ottobackwards) closes apache/metron-bro-plugin-kafka#32 +METRON-2013 The bro plugin docker scripts topic name should be configurable (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#27 +METRON-2020 Running run_end_to_end.sh with docker give warning if bash 4.0 (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#31 +METRON-1991 Bro plugin docker scripts should exit nonzero when bro and kafka counts differ (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#29 +METRON-2017 The Bro plugin docker data processing script incorrectly runs bro (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#30 +METRON-1990 Bro plugin docker should exit nonzero if it encounters issues (JonZeolla) closes apache/metron-bro-plugin-kafka#28 +METRON-2004 Bro plugin kafka docker_execute_shell.sh workdir should be unspecified (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#25 +METRON-2000 Fix bro plugin docker line counting for BRO_COUNT (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#24 +METRON-1992 Support sending a log to multiple topics (JonZeolla) closes apache/metron-bro-plugin-kafka#23 +METRON-1910 bro plugin segfaults on src/KafkaWriter.cc:72 (JonZeolla) closes apache/metron-bro-plugin-kafka#20 +METRON-1911 Create Docker based test environment for Bro Kafka Plugin (ottobackwards) closes apache/metron-bro-plugin-kafka#21 +METRON-1885 Remove version from bro plugin btest (JonZeolla) closes apache/metron-bro-plugin-kafka#19 +METRON-1827 Update librdkafka in metron-bro-plugin-kafka (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#13 +METRON-1866 Improve metron-bro-plugin-kafka documentation (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#17 +METRON-1304 Allow metron-bro-plugin-kafka to include or exclude logs (JonZeolla via nickwallen) closes apache/metron-bro-plugin-kafka#2 +METRON-1865 Fix metron-bro-plugin-kafka tests (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#16 +METRON-1828 Improve bro plugin contributing documentation (JonZeolla) closes apache/metron-bro-plugin-kafka#14 +METRON-1818 Remove config_files from bro-pkg.meta (JonZeolla) closes apache/metron-bro-plugin-kafka#11 +METRON-1800 Increment metron-bro-plugin-kafka version (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#10 +METRON-1773 Bro plugin docs should refer to Apache Metron project (nickwallen) closes apache/metron-bro-plugin-kafka#9 Added: dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/COPYING == --- dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/COPYING (added) +++ dev/metron/metron-bro-plugin-kafka/0.3.0-RC3/COPYING Thu Oct 10 18:27:50 2019 @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 +http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1
[metron-bro-plugin-kafka] branch Metron-bro-plugin-kafka_0.3.0 updated (e70500c -> abbbc9b)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to branch Metron-bro-plugin-kafka_0.3.0 in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git. from e70500c METRON-2069 Add btests for bro plugin topic_name selection (ottobackwards) closes apache/metron-bro-plugin-kafka#36 add abbbc9b METRON-2269 Cannot run Docker tests if src is not a git repo (ottobackwards) closes apache/metron-bro-plugin-kafka#37 No new revisions were added by this update. Summary of changes: docker/run_end_to_end.sh | 35 --- 1 file changed, 32 insertions(+), 3 deletions(-)
[metron-bro-plugin-kafka] tag apache-metron-bro-plugin-kafka_0.3.0-rc2 created (now e70500c)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to tag apache-metron-bro-plugin-kafka_0.3.0-rc2 in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git. at e70500c (commit) No new revisions were added by this update.
svn commit: r36128 - /dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/
Author: leet Date: Mon Sep 30 13:44:02 2019 New Revision: 36128 Log: Adding artifacts for metron-bro-plugin-kafka 0.3.0-RC2 Added: dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/ dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/CHANGES dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/COPYING dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/NOTICE dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/apache-metron-bro-plugin-kafka_0.3.0-rc2.tar.gz (with props) dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/apache-metron-bro-plugin-kafka_0.3.0-rc2.tar.gz.asc dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/apache-metron-bro-plugin-kafka_0.3.0-rc2.tar.gz.sha256 dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/apache-metron-bro-plugin-kafka_0.3.0-rc2.tar.gz.sha512 Added: dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/CHANGES == --- dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/CHANGES (added) +++ dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/CHANGES Mon Sep 30 13:44:02 2019 @@ -0,0 +1,24 @@ +METRON-2069 Add btests for bro plugin topic_name selection (ottobackwards) closes apache/metron-bro-plugin-kafka#36 +METRON-2045 Pass a version argument to the bro plugin docker scripts (JonZeolla) closes apache/metron-bro-plugin-kafka#35 +METRON-2003 Bro plugin topic should fall back to the log writer's path (JonZeolla) closes apache/metron-bro-plugin-kafka#26 +METRON-2025 Bro Kafka Plugin Docker should yum clean (ottobackwards) closes apache/metron-bro-plugin-kafka#33 +METRON-2021 Add screen to bro docker image (ottobackwards) closes apache/metron-bro-plugin-kafka#32 +METRON-2013 The bro plugin docker scripts topic name should be configurable (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#27 +METRON-2020 Running run_end_to_end.sh with docker give warning if bash 4.0 (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#31 +METRON-1991 Bro plugin docker scripts should exit nonzero when bro and kafka counts differ (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#29 +METRON-2017 The Bro plugin docker data processing script incorrectly runs bro (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#30 +METRON-1990 Bro plugin docker should exit nonzero if it encounters issues (JonZeolla) closes apache/metron-bro-plugin-kafka#28 +METRON-2004 Bro plugin kafka docker_execute_shell.sh workdir should be unspecified (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#25 +METRON-2000 Fix bro plugin docker line counting for BRO_COUNT (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#24 +METRON-1992 Support sending a log to multiple topics (JonZeolla) closes apache/metron-bro-plugin-kafka#23 +METRON-1910 bro plugin segfaults on src/KafkaWriter.cc:72 (JonZeolla) closes apache/metron-bro-plugin-kafka#20 +METRON-1911 Create Docker based test environment for Bro Kafka Plugin (ottobackwards) closes apache/metron-bro-plugin-kafka#21 +METRON-1885 Remove version from bro plugin btest (JonZeolla) closes apache/metron-bro-plugin-kafka#19 +METRON-1827 Update librdkafka in metron-bro-plugin-kafka (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#13 +METRON-1866 Improve metron-bro-plugin-kafka documentation (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#17 +METRON-1304 Allow metron-bro-plugin-kafka to include or exclude logs (JonZeolla via nickwallen) closes apache/metron-bro-plugin-kafka#2 +METRON-1865 Fix metron-bro-plugin-kafka tests (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#16 +METRON-1828 Improve bro plugin contributing documentation (JonZeolla) closes apache/metron-bro-plugin-kafka#14 +METRON-1818 Remove config_files from bro-pkg.meta (JonZeolla) closes apache/metron-bro-plugin-kafka#11 +METRON-1800 Increment metron-bro-plugin-kafka version (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#10 +METRON-1773 Bro plugin docs should refer to Apache Metron project (nickwallen) closes apache/metron-bro-plugin-kafka#9 Added: dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/COPYING == --- dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/COPYING (added) +++ dev/metron/metron-bro-plugin-kafka/0.3.0-RC2/COPYING Mon Sep 30 13:44:02 2019 @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 +http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by +
[metron-bro-plugin-kafka] branch Metron-bro-plugin-kafka_0.3.0 updated (667922f -> e70500c)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to branch Metron-bro-plugin-kafka_0.3.0 in repository https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git. from 667922f METRON-1885 Remove version from bro plugin btest (JonZeolla) closes apache/metron-bro-plugin-kafka#19 add 04593a6 METRON-1911 Create Docker based test environment for Bro Kafka Plugin (ottobackwards) closes apache/metron-bro-plugin-kafka#21 add b360b85 METRON-1910 bro plugin segfaults on src/KafkaWriter.cc:72 (JonZeolla) closes apache/metron-bro-plugin-kafka#20 add d96568e METRON-1992 Support sending a log to multiple topics (JonZeolla) closes apache/metron-bro-plugin-kafka#23 add bfc9cbb METRON-2000 Fix bro plugin docker line counting for BRO_COUNT (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#24 add d5f2562 METRON-2004 Bro plugin kafka docker_execute_shell.sh workdir should be unspecified (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#25 add 2655f71 METRON-1990 Bro plugin docker should exit nonzero if it encounters issues (JonZeolla) closes apache/metron-bro-plugin-kafka#28 add ddfba5c METRON-2017 The Bro plugin docker data processing script incorrectly runs bro (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#30 add 2f1edcb METRON-1991 Bro plugin docker scripts should exit nonzero when bro and kafka counts differ (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#29 add 946e664 METRON-2020 Running run_end_to_end.sh with docker give warning if bash 4.0 (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#31 add c497078 METRON-2013 The bro plugin docker scripts topic name should be configurable (JonZeolla via ottobackwards) closes apache/metron-bro-plugin-kafka#27 add 3cbc785 METRON-2021 Add screen to bro docker image (ottobackwards) closes apache/metron-bro-plugin-kafka#32 add 1d62215 METRON-2025 Bro Kafka Plugin Docker should yum clean (ottobackwards) closes apache/metron-bro-plugin-kafka#33 add 2f89267 METRON-2003 Bro plugin topic should fall back to the log writer's path (JonZeolla) closes apache/metron-bro-plugin-kafka#26 add 43c9166 METRON-2045 Pass a version argument to the bro plugin docker scripts (JonZeolla) closes apache/metron-bro-plugin-kafka#35 add e70500c METRON-2069 Add btests for bro plugin topic_name selection (ottobackwards) closes apache/metron-bro-plugin-kafka#36 No new revisions were added by this update. Summary of changes: .gitignore | 9 + CMakeLists.txt | 1 + NOTICE | 4 +- README.md | 41 ++- docker/README.md | 305 + .../containers/bro-localbuild-container/.screenrc | 22 ++ .../containers/bro-localbuild-container/Dockerfile | 56 docker/data/.gitignore | 4 + .../__load__.bro => docker/finish_end_to_end.sh| 22 +- docker/in_docker_scripts/build_bro_plugin.sh | 107 docker/in_docker_scripts/configure_bro_plugin.sh | 47 docker/in_docker_scripts/process_data_file.sh | 81 ++ docker/in_docker_scripts/wait-for-it.sh| 162 +++ .../in_docker_scripts/wait_for_kafka.sh| 18 +- .../in_docker_scripts/wait_for_zk.sh | 18 +- Makefile => docker/remove_timeout_message.sh | 57 ++-- docker/run_end_to_end.sh | 264 ++ docker/scripts/analyze_results.sh | 214 +++ docker/scripts/build_container.sh | 113 docker/scripts/cleanup_docker.sh | 91 ++ docker/scripts/create_docker_network.sh| 79 ++ docker/scripts/destroy_docker_network.sh | 76 + docker/scripts/docker_execute_build_bro_plugin.sh | 101 +++ .../scripts/docker_execute_configure_bro_plugin.sh | 84 ++ docker/scripts/docker_execute_process_data_file.sh | 113 docker/scripts/docker_execute_shell.sh | 79 ++ docker/scripts/docker_run_bro_container.sh | 168 docker/scripts/docker_run_consume_kafka.sh | 100 +++ docker/scripts/docker_run_create_topic_in_kafka.sh | 88 ++ docker/scripts/docker_run_get_offset_kafka.sh | 89 ++ docker/scripts/docker_run_kafka_container.sh | 82 ++ docker/scripts/docker_run_wait_for_kafka.sh| 82 ++ docker/scripts/docker_run_wait_for_zookeeper.sh| 82 ++ docker/scripts/docker_run_zookeeper_container.sh | 82 ++ docker/scripts/download_sample_pcaps.sh| 112 docker/scripts/print_results.sh| 92 +++ docker/s
[metron] branch master updated: METRON-2113 Update version to 0.7.2 (justinleet) closes apache/metron#1407
This is an automated email from the ASF dual-hosted git repository. leet pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron.git The following commit(s) were added to refs/heads/master by this push: new a6281d3 METRON-2113 Update version to 0.7.2 (justinleet) closes apache/metron#1407 a6281d3 is described below commit a6281d30ae82216dfe2c37576884a67b72e74ab7 Author: justinleet AuthorDate: Tue May 21 10:48:01 2019 -0400 METRON-2113 Update version to 0.7.2 (justinleet) closes apache/metron#1407 --- metron-analytics/metron-maas-common/pom.xml | 2 +- metron-analytics/metron-maas-service/pom.xml | 2 +- metron-analytics/metron-profiler-client/pom.xml | 2 +- metron-analytics/metron-profiler-common/pom.xml | 2 +- metron-analytics/metron-profiler-repl/pom.xml | 2 +- metron-analytics/metron-profiler-spark/pom.xml| 2 +- metron-analytics/metron-profiler-storm/pom.xml| 2 +- metron-analytics/metron-statistics/pom.xml| 2 +- metron-analytics/pom.xml | 2 +- metron-contrib/metron-docker/pom.xml | 2 +- metron-contrib/metron-performance/pom.xml | 2 +- metron-contrib/pom.xml| 2 +- metron-deployment/amazon-ec2/conf/defaults.yml| 2 +- metron-deployment/ansible/playbooks/docker_probe_install.yml | 2 +- metron-deployment/ansible/roles/ambari_master/defaults/main.yml | 2 +- .../development/centos6/ansible/inventory/group_vars/all | 2 +- .../development/ubuntu14/ansible/inventory/group_vars/all | 2 +- metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml| 4 ++-- metron-deployment/packaging/ambari/metron-mpack/pom.xml | 4 ++-- .../packaging/ambari/metron-mpack/src/main/resources/mpack.json | 2 +- metron-deployment/packaging/docker/deb-docker/pom.xml | 4 ++-- metron-deployment/packaging/docker/rpm-docker/pom.xml | 4 ++-- metron-deployment/pom.xml | 2 +- metron-interface/metron-alerts/package-lock.json | 2 +- metron-interface/metron-alerts/package.json | 2 +- metron-interface/metron-alerts/pom.xml| 2 +- metron-interface/metron-alerts/scripts/package.json | 2 +- metron-interface/metron-config/package-lock.json | 2 +- metron-interface/metron-config/package.json | 2 +- metron-interface/metron-config/pom.xml| 2 +- metron-interface/metron-config/scripts/package.json | 2 +- metron-interface/metron-rest-client/pom.xml | 2 +- metron-interface/metron-rest/pom.xml | 2 +- metron-interface/metron-rest/src/main/resources/application.yml | 2 +- metron-interface/pom.xml | 2 +- metron-platform/Performance-tuning-guide.md | 2 +- metron-platform/README.md | 2 +- metron-platform/elasticsearch-shaded/pom.xml | 2 +- .../metron-common-streaming/metron-common-storm/pom.xml | 2 +- metron-platform/metron-common-streaming/pom.xml | 2 +- metron-platform/metron-common/pom.xml | 2 +- metron-platform/metron-common/src/main/scripts/cluster_info.py| 2 +- metron-platform/metron-data-management/pom.xml| 2 +- metron-platform/metron-elasticsearch/pom.xml | 2 +- .../metron-enrichment/metron-enrichment-common/pom.xml| 2 +- metron-platform/metron-enrichment/metron-enrichment-storm/pom.xml | 2 +- metron-platform/metron-enrichment/pom.xml | 2 +- metron-platform/metron-hbase-client/pom.xml | 2 +- metron-platform/metron-hbase-server/pom.xml | 2 +- metron-platform/metron-hbase/pom.xml | 2 +- metron-platform/metron-indexing/pom.xml | 2 +- metron-platform/metron-integration-test/pom.xml | 2 +- metron-platform/metron-job/pom.xml| 2 +- metron-platform/metron-management/pom.xml | 2 +- metron-platform/metron-parsing/metron-parsers-common/pom.xml | 2 +- metron-platform/metron-parsing/metron-parsers/pom.xml | 2 +- metron-platform/metron-parsing/metron-parsing-storm/pom.xml | 2 +- metron-platform/metron-parsing/pom.xml| 2 +- metron-platform/metron-pcap-backend/pom.xml | 2 +- .../metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh
svn commit: r34063 - /release/metron/0.7.0/
Author: leet Date: Wed May 15 21:53:51 2019 New Revision: 34063 Log: Removing old 0.7.0 release Removed: release/metron/0.7.0/
svn commit: r34036 - in /release/metron/0.7.1: ./ CHANGES LICENSE NOTICE apache-metron_0.7.1.tar.gz apache-metron_0.7.1.tar.gz.asc apache-metron_0.7.1.tar.gz.sha256 apache-metron_0.7.1.tar.gz.sha512
Author: leet Date: Tue May 14 13:10:45 2019 New Revision: 34036 Log: Adding artifacts for Metron 0.7.1 Added: release/metron/0.7.1/ release/metron/0.7.1/CHANGES release/metron/0.7.1/LICENSE release/metron/0.7.1/NOTICE release/metron/0.7.1/apache-metron_0.7.1.tar.gz (with props) release/metron/0.7.1/apache-metron_0.7.1.tar.gz.asc release/metron/0.7.1/apache-metron_0.7.1.tar.gz.sha256 release/metron/0.7.1/apache-metron_0.7.1.tar.gz.sha512 Added: release/metron/0.7.1/CHANGES == --- release/metron/0.7.1/CHANGES (added) +++ release/metron/0.7.1/CHANGES Tue May 14 13:10:45 2019 @@ -0,0 +1,93 @@ +METRON-2100 Update developer documentation for full dev management UI parser aggregation feature gap (mmiklavc) closes apache/metron#1398 +METRON-2018 Update prepare-commit to add Bro plugin tests (ottobackwards) closes apache/metron#1348 +METRON-2093 Metron RC check script is outdated (justinleet) closes apache/metron#1394 +METRON-2094 Create CentOS 7 Development Environment (nickwallen) closes apache/metron#1395 +METRON-2090 Full dev is failing with missing org.mortbay.jetty:jetty-util:jar:6.1.26.hwx dependency (merrimanr) closes apache/metron#1391 +METRON-2091 SimpleHBaseEnrichmentWriterTest should be included in tests (merrimanr) closes apache/metron#1392 +METRON-2078 Remove Storm dependency from metron-writer (merrimanr) closes apache/metron#1386 +METRON-2065 Setting Parser Output Topic in Sensor Config is broken (merrimanr) closes apache/metron#1377 +METRON-2067 Maven pom file duplicate dependency fixes (mmiklavc) closes apache/metron#1379 +METRON-2074 Script to handle TGT renewal with Storm and Kerberos enabled (mmiklavc) closes apache/metron#1382 +METRON-2082 Update the README document steps to run Batch Profiler (MohanDV via nickwallen) closes apache/metron#1387 +METRON-2006 Reenable JacoCo code coverage (justinleet via nickwallen) closes apache/metron#1339 +METRON-2071 Add MAP_PUT and MAP_MERGE to Stellar (mmiklavc) closes apache/metron#1385 +METRON-2014 Add architectural documentation for metron-writer (merrimanr) closes apache/metron#1381 +METRON-2026 Remove Storm dependency from metron-common (merrimanr) closes apache/metron#1351 +METRON-2062 Metron Alerts: Accidentally commited 'fdescribe' in unit tests (ruffle1986 via mmiklavc) closes apache/metron#1372 +METRON-2050 Automatically populate a list of enrichments from HBase (mmiklavc) closes apache/metron#1365 +METRON-2060 Improving Alerts table config pane (tiborm via mmiklavc) closes apache/metron#1375 +METRON-2064 Metron REST API overwriting global.json values (merrimanr) closes apache/metron#1376 +METRON-2066 Documentation and logging corrections (mmiklavc) closes apache/metron#1378 +METRON-1654 findOne request after an alert patch returns with the original state of the alert item (merrimanr) closes apache/metron#1344 +METRON-2053 Refactor metron-enrichment to decouple Storm dependencies (mmiklavc) closes apache/metron#1368 +METRON-2022 Metron rest creates large number of connections to ZK which causes subsequent connection to zk fail (merrimanr) closes apache/metron#1367 +METRON-2056 Support LDAP Bind Authentication (nickwallen) closes apache/metron#1371 +METRON-2039 Time range queries do not work with Solr (merrimanr) closes apache/metron#1359 +METRON-2052 UI Changing default query time range to 15 minutes (tiborm via sardell) closes apache/metron#1369 +METRON-2051 Improve stellar-zeppelin documentation (mmiklavc) closes apache/metron#1366 +METRON-2023 UI Covering Grok Parser Creation with Cypress tests (tiborm via sardell) closes apache/metron#1364 +METRON-2046 IS_EMPTY stellar functions fails on empty map (anandsubbu) closes apache/metron#1363 +METRON-2029 Configure Table should have filter (sardell) closes apache/metron#1356 +METRON-2032 Create summary table having list of stellar functions in README (anandsubbu) closes apache/metron#1354 +METRON-2038 Enrichment Loader Fails When Run as MR Job (nickwallen) closes apache/metron#1358 +METRON-2035 Allow User to Configure Role Names for Access Control (nickwallen) closes apache/metron#1355 +METRON-2041 RegularExpressionsParser in wrong source folder (mmiklavc) closes apache/metron#1361 +METRON-2036 Maven builds fail locally in HDFSWriterTest (mmiklavc) closes apache/metron#1357 +METRON-2030 SensorParserGroupControllerIntegrationTest intermittent errors (merrimanr via mmiklavc) closes apache/metron#1352 +METRON-2031 UI Turning off initial search request and polling by default on Alerts UI (tiborm via mmiklavc) closes apache/metron#1353 +METRON-2012 Unable to Execute Stellar Functions Against HBase in the REPL (nickwallen) closes apache/metron#1345 +METRON-1971 Short timeout value in Cypress may cause
[metron] tag apache-metron_0.7.1-release created (now e1d1901)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to tag apache-metron_0.7.1-release in repository https://gitbox.apache.org/repos/asf/metron.git. at e1d1901 (commit) No new revisions were added by this update.
svn commit: r33955 - in /dev/metron/0.7.1-RC2: ./ CHANGES LICENSE NOTICE apache-metron_0.7.1-rc2.tar.gz apache-metron_0.7.1-rc2.tar.gz.asc apache-metron_0.7.1-rc2.tar.gz.sha256 apache-metron_0.7.1-rc2
Author: leet Date: Wed May 8 22:46:16 2019 New Revision: 33955 Log: Adding artifacts for metron 0.7.1-RC2 Added: dev/metron/0.7.1-RC2/ dev/metron/0.7.1-RC2/CHANGES dev/metron/0.7.1-RC2/LICENSE dev/metron/0.7.1-RC2/NOTICE dev/metron/0.7.1-RC2/apache-metron_0.7.1-rc2.tar.gz (with props) dev/metron/0.7.1-RC2/apache-metron_0.7.1-rc2.tar.gz.asc dev/metron/0.7.1-RC2/apache-metron_0.7.1-rc2.tar.gz.sha256 dev/metron/0.7.1-RC2/apache-metron_0.7.1-rc2.tar.gz.sha512 Added: dev/metron/0.7.1-RC2/CHANGES == --- dev/metron/0.7.1-RC2/CHANGES (added) +++ dev/metron/0.7.1-RC2/CHANGES Wed May 8 22:46:16 2019 @@ -0,0 +1,93 @@ +METRON-2100 Update developer documentation for full dev management UI parser aggregation feature gap (mmiklavc) closes apache/metron#1398 +METRON-2018 Update prepare-commit to add Bro plugin tests (ottobackwards) closes apache/metron#1348 +METRON-2093 Metron RC check script is outdated (justinleet) closes apache/metron#1394 +METRON-2094 Create CentOS 7 Development Environment (nickwallen) closes apache/metron#1395 +METRON-2090 Full dev is failing with missing org.mortbay.jetty:jetty-util:jar:6.1.26.hwx dependency (merrimanr) closes apache/metron#1391 +METRON-2091 SimpleHBaseEnrichmentWriterTest should be included in tests (merrimanr) closes apache/metron#1392 +METRON-2078 Remove Storm dependency from metron-writer (merrimanr) closes apache/metron#1386 +METRON-2065 Setting Parser Output Topic in Sensor Config is broken (merrimanr) closes apache/metron#1377 +METRON-2067 Maven pom file duplicate dependency fixes (mmiklavc) closes apache/metron#1379 +METRON-2074 Script to handle TGT renewal with Storm and Kerberos enabled (mmiklavc) closes apache/metron#1382 +METRON-2082 Update the README document steps to run Batch Profiler (MohanDV via nickwallen) closes apache/metron#1387 +METRON-2006 Reenable JacoCo code coverage (justinleet via nickwallen) closes apache/metron#1339 +METRON-2071 Add MAP_PUT and MAP_MERGE to Stellar (mmiklavc) closes apache/metron#1385 +METRON-2014 Add architectural documentation for metron-writer (merrimanr) closes apache/metron#1381 +METRON-2026 Remove Storm dependency from metron-common (merrimanr) closes apache/metron#1351 +METRON-2062 Metron Alerts: Accidentally commited 'fdescribe' in unit tests (ruffle1986 via mmiklavc) closes apache/metron#1372 +METRON-2050 Automatically populate a list of enrichments from HBase (mmiklavc) closes apache/metron#1365 +METRON-2060 Improving Alerts table config pane (tiborm via mmiklavc) closes apache/metron#1375 +METRON-2064 Metron REST API overwriting global.json values (merrimanr) closes apache/metron#1376 +METRON-2066 Documentation and logging corrections (mmiklavc) closes apache/metron#1378 +METRON-1654 findOne request after an alert patch returns with the original state of the alert item (merrimanr) closes apache/metron#1344 +METRON-2053 Refactor metron-enrichment to decouple Storm dependencies (mmiklavc) closes apache/metron#1368 +METRON-2022 Metron rest creates large number of connections to ZK which causes subsequent connection to zk fail (merrimanr) closes apache/metron#1367 +METRON-2056 Support LDAP Bind Authentication (nickwallen) closes apache/metron#1371 +METRON-2039 Time range queries do not work with Solr (merrimanr) closes apache/metron#1359 +METRON-2052 UI Changing default query time range to 15 minutes (tiborm via sardell) closes apache/metron#1369 +METRON-2051 Improve stellar-zeppelin documentation (mmiklavc) closes apache/metron#1366 +METRON-2023 UI Covering Grok Parser Creation with Cypress tests (tiborm via sardell) closes apache/metron#1364 +METRON-2046 IS_EMPTY stellar functions fails on empty map (anandsubbu) closes apache/metron#1363 +METRON-2029 Configure Table should have filter (sardell) closes apache/metron#1356 +METRON-2032 Create summary table having list of stellar functions in README (anandsubbu) closes apache/metron#1354 +METRON-2038 Enrichment Loader Fails When Run as MR Job (nickwallen) closes apache/metron#1358 +METRON-2035 Allow User to Configure Role Names for Access Control (nickwallen) closes apache/metron#1355 +METRON-2041 RegularExpressionsParser in wrong source folder (mmiklavc) closes apache/metron#1361 +METRON-2036 Maven builds fail locally in HDFSWriterTest (mmiklavc) closes apache/metron#1357 +METRON-2030 SensorParserGroupControllerIntegrationTest intermittent errors (merrimanr via mmiklavc) closes apache/metron#1352 +METRON-2031 UI Turning off initial search request and polling by default on Alerts UI (tiborm via mmiklavc) closes apache/metron#1353 +METRON-2012 Unable to Execute Stellar Functions Against HBase in the REPL (nickwallen) closes apache/metron#1345 +METRON-1971 Short timeout value
[metron] tag apache-metron_0.7.1-rc2 created (now e1d1901)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to tag apache-metron_0.7.1-rc2 in repository https://gitbox.apache.org/repos/asf/metron.git. at e1d1901 (commit) No new revisions were added by this update.
[metron] branch master updated: METRON-2093 Metron RC check script is outdated (justinleet) closes apache/metron#1394
This is an automated email from the ASF dual-hosted git repository.
leet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 4390e38 METRON-2093 Metron RC check script is outdated (justinleet)
closes apache/metron#1394
4390e38 is described below
commit 4390e38c2460624734ead7ab6f9fff3738471c4e
Author: justinleet
AuthorDate: Thu May 2 11:35:27 2019 -0400
METRON-2093 Metron RC check script is outdated (justinleet) closes
apache/metron#1394
---
dev-utilities/release-utils/metron-rc-check | 21 +++--
.../release-utils/prepare-release-candidate | 3 +--
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/dev-utilities/release-utils/metron-rc-check
b/dev-utilities/release-utils/metron-rc-check
index 143ba85..5b3a5d9 100755
--- a/dev-utilities/release-utils/metron-rc-check
+++ b/dev-utilities/release-utils/metron-rc-check
@@ -30,7 +30,9 @@ function help {
echo " "
}
-METRON_DIST="https://dist.apache.org/repos/dist/dev/metron/;
+APACHE_REPO="https://dist.apache.org/repos/dist/;
+METRON_DIST=${APACHE_REPO}"dev/metron/"
+METRON_KEYS=${APACHE_REPO}"release/metron/KEYS"
# print help, if the user just runs this without any args
if [ "$#" -eq 0 ]; then
help
@@ -156,14 +158,13 @@ if [ ! -d "$WORK" ]; then
fi
echo "Working directory $WORK"
-KEYS="$METRON_RC_DIST/KEYS"
-METRON_ASSEMBLY="$METRON_RC_DIST/apache-metron-$METRON_VERSION-$RC.tar.gz"
+METRON_ASSEMBLY="$METRON_RC_DIST/apache-metron_$METRON_VERSION-$RC.tar.gz"
METRON_ASSEMBLY_SIG="$METRON_ASSEMBLY.asc"
-echo "Downloading $KEYS"
-if ! wget -P "$WORK" "$KEYS" ; then
- echo "[ERROR] Failed to download $KEYS"
+echo "Downloading $METRON_KEYS"
+if ! wget -P "$WORK" "$METRON_KEYS" ; then
+ echo "[ERROR] Failed to download $METRON_KEYS"
exit 1
fi
@@ -206,7 +207,7 @@ if ! gpg --import KEYS ; then
fi
echo "Verifying Metron Assembly"
-if ! gpg --verify ./"apache-metron-$METRON_VERSION-$RC.tar.gz.asc"
"apache-metron-$METRON_VERSION-$RC.tar.gz" ; then
+if ! gpg --verify ./"apache-metron_$METRON_VERSION-$RC.tar.gz.asc"
"apache-metron_$METRON_VERSION-$RC.tar.gz" ; then
echo "[ERROR] failed to verify Metron Assembly"
exit 1
fi
@@ -227,7 +228,7 @@ if [ -n "$BRO" ]; then
fi
echo "Unpacking Assemblies"
-if ! tar -xzf "apache-metron-$METRON_VERSION-$RC.tar.gz" ; then
+if ! tar -xzf "apache-metron_$METRON_VERSION-$RC.tar.gz" ; then
echo "[ERROR] failed to unpack Metron Assembly"
exit 1
fi
@@ -238,7 +239,7 @@ read -p " run test suite [install, unit tests, integration
tests, ui tests, lic
echo
DID_BUILD=0
if [[ $REPLY =~ ^[Yy]$ ]]; then
- cd "apache-metron-$METRON_VERSION-$RC" || exit 1
+ cd "apache-metron_${METRON_VERSION}-$RC" || exit 1
if ! mvn -q -T 2C -DskipTests clean install ; then
echo "[ERROR] failed to mvn install metron"
exit 1
@@ -273,7 +274,7 @@ echo ""
read -p " run vagrant full_dev? [yN] " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
- cd
"$WORK/apache-metron-$METRON_VERSION-$RC/metron-deployment/development/centos6"
|| exit 1
+ cd
"$WORK/apache-metron_${METRON_VERSION}-$RC/metron-deployment/development/centos6"
|| exit 1
if [[ ${DID_BUILD} -ne 1 ]]; then
vagrant up
else
diff --git a/dev-utilities/release-utils/prepare-release-candidate
b/dev-utilities/release-utils/prepare-release-candidate
index cb0585d..e2daa4d 100755
--- a/dev-utilities/release-utils/prepare-release-candidate
+++ b/dev-utilities/release-utils/prepare-release-candidate
@@ -299,7 +299,7 @@ if [[ $? -ne 0 ]]; then
exit 1
fi
-printf "Extracting LICENSE, NOTICE, and KEYS from tarball\n" # Only pull from
core
+printf "Extracting LICENSE and NOTICE from tarball\n" # Only pull from core
cd ${ART_DIR}
if [ "${CHOSEN_REPO}" = "${BRO_PLUGIN_REPO_NAME}" ]; then
@@ -311,7 +311,6 @@ fi
# TODO figure out what to do for bro repo here. The KEYS file only needs to
live in the /dist root, rather than in each sub repo.
# Should we have a separate process for adding to the KEYS file without doing
a Metron release?
-#tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/KEYS"
tar --strip-components=1 -zxvf "${ARTIFACT}.tar.gz" "${ARTIFACT}/NOTICE"
# Add the directory and commit to subversion
[metron] tag apache-metron_0.7.1-rc1 created (now f99cc54)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to tag apache-metron_0.7.1-rc1 in repository https://gitbox.apache.org/repos/asf/metron.git. at f99cc54 (commit) No new revisions were added by this update.
svn commit: r33780 - in /dev/metron/0.7.1-RC1: ./ CHANGES LICENSE NOTICE apache-metron_0.7.1-rc1.tar.gz apache-metron_0.7.1-rc1.tar.gz.asc apache-metron_0.7.1-rc1.tar.gz.sha256 apache-metron_0.7.1-rc1
Author: leet Date: Thu Apr 25 18:51:34 2019 New Revision: 33780 Log: Adding artifacts for metron 0.7.1-RC1 Added: dev/metron/0.7.1-RC1/ dev/metron/0.7.1-RC1/CHANGES dev/metron/0.7.1-RC1/LICENSE dev/metron/0.7.1-RC1/NOTICE dev/metron/0.7.1-RC1/apache-metron_0.7.1-rc1.tar.gz (with props) dev/metron/0.7.1-RC1/apache-metron_0.7.1-rc1.tar.gz.asc dev/metron/0.7.1-RC1/apache-metron_0.7.1-rc1.tar.gz.sha256 dev/metron/0.7.1-RC1/apache-metron_0.7.1-rc1.tar.gz.sha512 Added: dev/metron/0.7.1-RC1/CHANGES == --- dev/metron/0.7.1-RC1/CHANGES (added) +++ dev/metron/0.7.1-RC1/CHANGES Thu Apr 25 18:51:34 2019 @@ -0,0 +1,89 @@ +METRON-2090 Full dev is failing with missing org.mortbay.jetty:jetty-util:jar:6.1.26.hwx dependency (merrimanr) closes apache/metron#1391 +METRON-2091 SimpleHBaseEnrichmentWriterTest should be included in tests (merrimanr) closes apache/metron#1392 +METRON-2078 Remove Storm dependency from metron-writer (merrimanr) closes apache/metron#1386 +METRON-2065 Setting Parser Output Topic in Sensor Config is broken (merrimanr) closes apache/metron#1377 +METRON-2067 Maven pom file duplicate dependency fixes (mmiklavc) closes apache/metron#1379 +METRON-2074 Script to handle TGT renewal with Storm and Kerberos enabled (mmiklavc) closes apache/metron#1382 +METRON-2082 Update the README document steps to run Batch Profiler (MohanDV via nickwallen) closes apache/metron#1387 +METRON-2006 Reenable JacoCo code coverage (justinleet via nickwallen) closes apache/metron#1339 +METRON-2071 Add MAP_PUT and MAP_MERGE to Stellar (mmiklavc) closes apache/metron#1385 +METRON-2014 Add architectural documentation for metron-writer (merrimanr) closes apache/metron#1381 +METRON-2026 Remove Storm dependency from metron-common (merrimanr) closes apache/metron#1351 +METRON-2062 Metron Alerts: Accidentally commited 'fdescribe' in unit tests (ruffle1986 via mmiklavc) closes apache/metron#1372 +METRON-2050 Automatically populate a list of enrichments from HBase (mmiklavc) closes apache/metron#1365 +METRON-2060 Improving Alerts table config pane (tiborm via mmiklavc) closes apache/metron#1375 +METRON-2064 Metron REST API overwriting global.json values (merrimanr) closes apache/metron#1376 +METRON-2066 Documentation and logging corrections (mmiklavc) closes apache/metron#1378 +METRON-1654 findOne request after an alert patch returns with the original state of the alert item (merrimanr) closes apache/metron#1344 +METRON-2053 Refactor metron-enrichment to decouple Storm dependencies (mmiklavc) closes apache/metron#1368 +METRON-2022 Metron rest creates large number of connections to ZK which causes subsequent connection to zk fail (merrimanr) closes apache/metron#1367 +METRON-2056 Support LDAP Bind Authentication (nickwallen) closes apache/metron#1371 +METRON-2039 Time range queries do not work with Solr (merrimanr) closes apache/metron#1359 +METRON-2052 UI Changing default query time range to 15 minutes (tiborm via sardell) closes apache/metron#1369 +METRON-2051 Improve stellar-zeppelin documentation (mmiklavc) closes apache/metron#1366 +METRON-2023 UI Covering Grok Parser Creation with Cypress tests (tiborm via sardell) closes apache/metron#1364 +METRON-2046 IS_EMPTY stellar functions fails on empty map (anandsubbu) closes apache/metron#1363 +METRON-2029 Configure Table should have filter (sardell) closes apache/metron#1356 +METRON-2032 Create summary table having list of stellar functions in README (anandsubbu) closes apache/metron#1354 +METRON-2038 Enrichment Loader Fails When Run as MR Job (nickwallen) closes apache/metron#1358 +METRON-2035 Allow User to Configure Role Names for Access Control (nickwallen) closes apache/metron#1355 +METRON-2041 RegularExpressionsParser in wrong source folder (mmiklavc) closes apache/metron#1361 +METRON-2036 Maven builds fail locally in HDFSWriterTest (mmiklavc) closes apache/metron#1357 +METRON-2030 SensorParserGroupControllerIntegrationTest intermittent errors (merrimanr via mmiklavc) closes apache/metron#1352 +METRON-2031 UI Turning off initial search request and polling by default on Alerts UI (tiborm via mmiklavc) closes apache/metron#1353 +METRON-2012 Unable to Execute Stellar Functions Against HBase in the REPL (nickwallen) closes apache/metron#1345 +METRON-1971 Short timeout value in Cypress may cause build failures (sardell) closes apache/metron#1323 +METRON-1940 Check if not and install Elastic search templates / Solr collections when indexing server is restarted (MohanDV) closes apache/metron#1305 +METRON-2019 Improve Metron REST Logging (merrimanr) closes apache/metron#1347 +METRON-2016 Parser aggregate groups should be persisted and available through REST (merrimanr) closes apache/metron#1346
[metron] branch Metron_0.7.1 created (now f99cc54)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to branch Metron_0.7.1 in repository https://gitbox.apache.org/repos/asf/metron.git. at f99cc54 METRON-2090 Full dev is failing with missing org.mortbay.jetty:jetty-util:jar:6.1.26.hwx dependency (merrimanr) closes apache/metron#1391 No new revisions were added by this update.
[metron] branch master updated: METRON-2005 Batch Writer writes 0-byte files to HDFS on rotation (justinleet) closes apache/metron#1338
This is an automated email from the ASF dual-hosted git repository.
leet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 5e9cf70 METRON-2005 Batch Writer writes 0-byte files to HDFS on
rotation (justinleet) closes apache/metron#1338
5e9cf70 is described below
commit 5e9cf705ef5feb1d723f584c1b6134bcc1eda9cf
Author: justinleet
AuthorDate: Tue Feb 19 14:09:02 2019 -0500
METRON-2005 Batch Writer writes 0-byte files to HDFS on rotation
(justinleet) closes apache/metron#1338
---
metron-platform/metron-writer/README.md| 3 +
metron-platform/metron-writer/pom.xml | 5 ++
.../writer/hdfs/ClonedSyncPolicyCreator.java | 3 +
.../org/apache/metron/writer/hdfs/HdfsWriter.java | 74 +--
.../apache/metron/writer/hdfs/SourceHandler.java | 25 +--
.../metron/writer/hdfs/SourceHandlerCallback.java | 23 +-
.../metron/writer/hdfs/SourceHandlerKey.java | 8 ++
.../apache/metron/writer/hdfs/HdfsWriterTest.java | 38 --
.../metron/writer/hdfs/SourceHandlerTest.java | 86 ++
9 files changed, 194 insertions(+), 71 deletions(-)
diff --git a/metron-platform/metron-writer/README.md
b/metron-platform/metron-writer/README.md
index bbec39b..ed4f053 100644
--- a/metron-platform/metron-writer/README.md
+++ b/metron-platform/metron-writer/README.md
@@ -65,6 +65,9 @@ To manage the output path, a base path argument is provided
by the Flux file, wi
This means that all output will land in `/apps/metron/`. With no further
adjustment, it will be `/apps/metron//`.
However, by modifying the sensor's JSON config, it is possible to provide
additional pathing based on the the message itself.
+The output format of a file will be
`{prefix}{componentId}-{taskId}-{rotationNum}-{timestamp}{extension}`. Notably,
because of the way
+file rotations are handled by the HdfsWriter, `rotationNum` will always be 0,
but RotationActions still get executed normally.
+
E.g.
```
{
diff --git a/metron-platform/metron-writer/pom.xml
b/metron-platform/metron-writer/pom.xml
index 0002f7a..a11ce6e 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/pom.xml
@@ -213,6 +213,11 @@
${project.parent.version}
test
+
+org.apache.metron
+stellar-common
+${project.parent.version}
+
diff --git
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/ClonedSyncPolicyCreator.java
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/ClonedSyncPolicyCreator.java
index 4d32fc9..1f908a9 100644
---
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/ClonedSyncPolicyCreator.java
+++
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/ClonedSyncPolicyCreator.java
@@ -37,6 +37,9 @@ public class ClonedSyncPolicyCreator implements
SyncPolicyCreator {
// SyncPolicy object does not implement Cloneable, so we'll need to
clone it via serialization
//to get a fresh policy object. Note: this would be expensive if it was
in the critical path,
// but should be called infrequently (once per sync).
+
+ // Reset the SyncPolicy to ensure that the new count properly resets.
+ syncPolicy.reset();
byte[] serializedForm = SerDeUtils.toBytes(syncPolicy);
return SerDeUtils.fromBytes(serializedForm, SyncPolicy.class);
}
diff --git
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
index 1ba9a6b..aaa58fa 100644
---
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
+++
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/hdfs/HdfsWriter.java
@@ -17,29 +17,37 @@
*/
package org.apache.metron.writer.hdfs;
+import java.io.IOException;
+import java.io.Serializable;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.metron.common.configuration.IndexingConfigurations;
+import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.writer.BulkMessageWriter;
+import org.apache.metron.common.writer.BulkWriterResponse;
+import org.apache.metron.stellar.common.StellarProcessor;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.MapVariableResolver;
import org.apache.metron.stellar.dsl.StellarFunctions;
import org.apache.metron.stellar.dsl.VariableResolver;
-import org.apache.metron.stellar.common.StellarProcessor;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.tuple.Tuple;
-import
[metron] branch master updated: METRON-1929 Build GET_ASN Stellar function (justinleet) closes apache/metron#1299
This is an automated email from the ASF dual-hosted git repository.
leet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new f43035c METRON-1929 Build GET_ASN Stellar function (justinleet)
closes apache/metron#1299
f43035c is described below
commit f43035c02ef01f07ff382bbf136eb1bada727fbb
Author: justinleet
AuthorDate: Thu Jan 17 12:59:58 2019 -0500
METRON-1929 Build GET_ASN Stellar function (justinleet) closes
apache/metron#1299
---
Upgrading.md | 8 +
dependencies_with_url.csv | 4 +-
.../compose/hadoop/docker-entrypoint.sh| 2 +-
.../packaging/ambari/metron-mpack/README.md| 4 +-
.../configuration/metron-enrichment-env.xml| 8 +-
.../CURRENT/package/scripts/enrichment_commands.py | 45 ++--
.../CURRENT/package/scripts/enrichment_master.py | 4 +-
.../CURRENT/package/scripts/params/params_linux.py | 4 +-
.../package/scripts/params/status_params.py| 2 +-
.../METRON/CURRENT/themes/metron_theme.json| 10 +
.../packaging/docker/rpm-docker/SPECS/metron.spec | 2 +-
.../metron-alerts/cypress/fixtures/config.json | 2 +-
metron-platform/metron-data-management/README.md | 19 +-
...tLoader.java => MaxmindDbEnrichmentLoader.java} | 110 ++---
...richment_load.sh => maxmind_enrichment_load.sh} | 2 +-
...est.java => MaxmindDbEnrichmentLoaderTest.java} | 67 --
metron-platform/metron-enrichment/pom.xml | 2 +-
.../metron/enrichment/adapters/geo/GeoAdapter.java | 9 +-
.../adapters/maxmind/MaxMindDatabase.java | 133 +++
.../adapters/maxmind/MaxMindDbUtilities.java | 127 ++
.../adapters/maxmind/asn/GeoLiteAsnDatabase.java | 164 +
.../geo/GeoLiteCityDatabase.java} | 169 +++--
.../{ => maxmind}/geo/hash/DistanceStrategies.java | 2 +-
.../{ => maxmind}/geo/hash/DistanceStrategy.java | 2 +-
.../{ => maxmind}/geo/hash/GeoHashUtil.java| 6 +-
.../enrichment/bolt/ThreatIntelJoinBolt.java | 10 +-
.../enrichment/bolt/UnifiedEnrichmentBolt.java | 8 +-
...tFunctions.java => AsnEnrichmentFunctions.java} | 73 +++---
.../enrichment/stellar/GeoEnrichmentFunctions.java | 20 +-
.../enrichment/stellar/GeoHashFunctions.java | 12 +-
.../adapters/geo/GeoLiteDatabaseTest.java | 190 ---
.../maxmind/asn/GeoLiteAsnDatabaseTest.java| 157
.../adapters/{ => maxmind}/geo/GeoAdapterTest.java | 8 +-
.../maxmind/geo/GeoLiteCityDatabaseTest.java | 263 +
.../enrichment/bolt/GenericEnrichmentBoltTest.java | 7 +-
.../enrichment/bolt/ThreatIntelJoinBoltTest.java | 9 +-
.../integration/EnrichmentIntegrationTest.java | 10 +-
.../stellar/AsnEnrichmentFunctionsTest.java| 173 ++
.../stellar/GeoEnrichmentFunctionsTest.java| 83 +--
.../resources/GeoLite/GeoIP2-City-Test-2.mmdb.gz | Bin 10011 -> 0 bytes
.../src/test/resources/GeoLite/GeoLite2-ASN.tar.gz | Bin 0 -> 3460335 bytes
...IP2-City-Test.mmdb.gz => GeoLite2-City.mmdb.gz} | Bin
.../test/resources/GeoLite/GeoLite2-City.tar.gz| Bin 0 -> 26788236 bytes
metron-stellar/stellar-common/README.md| 7 +
44 files changed, 1462 insertions(+), 475 deletions(-)
diff --git a/Upgrading.md b/Upgrading.md
index 1b2324b..fcd5f27 100644
--- a/Upgrading.md
+++ b/Upgrading.md
@@ -19,6 +19,14 @@ limitations under the License.
This document constitutes a per-version listing of changes of
configuration which are non-backwards compatible.
+## 0.7.0 to 0.7.1
+### [METRON-1929: Build GET_ASN Stellar
function](https://issues.apache.org/jira/browse/METRON-1929)
+The script for `geo_enrichment_load.sh` has been renamed, and now is
`maxmind_enrichment_load.sh`. A couple changes should happen for users who are
upgrading.
+
+* The MaxMind GeoLite2 ASN database should be loaded onto HDFS at
/apps/metron/asn/default/GeoLite2-ASN.tar.gz OR the global configuration
property `asn.hdfs.file` can be set to point to a custom HDFS location.
+* Any custom scripts or tasks that use this script should be updated. In
addition, this updated script also retrieves the GeoLite2 ASN database. The
`-ra` flag can be used to provide a custom location for this database if
offline install is needed. Otherwise, it will retrieve the latest from MaxMind.
+
+
## 0.6.0 to 0.7.0
### [METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST
API](https://issues.apache.org/jira/browse/METRON-1834)
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 5856a3a..e16650d 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -28,8 +28,8 @@ com.jayway.jsonpath:json-path:jar:2.3.0:compile,Apache
v2,https://gith
[metron] branch sparkPoc created (now aa85ffa)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to branch sparkPoc in repository https://gitbox.apache.org/repos/asf/metron.git. at aa85ffa METRON-1939 Update version to 0.7.1 (justinleet via nickwallen) closes apache/metron#1303 No new revisions were added by this update.
svn commit: r31577 - /release/metron/0.6.0/
Author: leet Date: Mon Dec 17 17:11:53 2018 New Revision: 31577 Log: Removing old 0.6.0 release Removed: release/metron/0.6.0/
svn commit: r31544 - in /release/metron/0.7.0: ./ CHANGES LICENSE NOTICE apache-metron-0.7.0.tar.gz apache-metron-0.7.0.tar.gz.asc apache-metron-0.7.0.tar.gz.sha256 apache-metron-0.7.0.tar.gz.sha512
Author: leet Date: Fri Dec 14 21:03:33 2018 New Revision: 31544 Log: Adding artifacts for Metron 0.7.0 Added: release/metron/0.7.0/ release/metron/0.7.0/CHANGES release/metron/0.7.0/LICENSE release/metron/0.7.0/NOTICE release/metron/0.7.0/apache-metron-0.7.0.tar.gz (with props) release/metron/0.7.0/apache-metron-0.7.0.tar.gz.asc release/metron/0.7.0/apache-metron-0.7.0.tar.gz.sha256 release/metron/0.7.0/apache-metron-0.7.0.tar.gz.sha512 Added: release/metron/0.7.0/CHANGES == --- release/metron/0.7.0/CHANGES (added) +++ release/metron/0.7.0/CHANGES Fri Dec 14 21:03:33 2018 @@ -0,0 +1,98 @@ +METRON-1928 Bump Metron version to 0.7.0 for release. (justinleet) closes apache/metron#1293 +METRON-1931 Update dev utilities to support new repo location (rlenferink via justinleet) closes apache/metron#1295 +METRON-1922 Escaping incorrectly handled in current aesh version (justinleet) closes apache/metron#1291 +METRON-1867 Remove `/api/v1/update/replace` endpoint (nickwallen) closes apache/metron#1284 +METRON-1810 Storm Profiler Intermittent Test Failure (nickwallen) closes apache/metron#1289 +METRON-1909 Remove http filter from release utils changelog generation (justinleet) closes apache/metron#1283 +METRON-1869 Unable to Sort an Escalated Meta Alert (nickwallen) closes apache/metron#1280 +METRON-1889: Add any missing timestamp fields to unified enrichment topology (mmiklavc via mmiklavc) closes apache/metron#1286 +METRON-1913 metron-alert UI - Build broken by missing transitive dependency (tiborm via sardell) closes apache/metron#1285 +METRON-1845 Correct Test Data Load in Elasticsearch Integration Tests (nickwallen) closes apache/metron#1247 +METRON-1888 Default Topology Settings in MPack Cause Profiler to Stall (nickwallen) closes apache/metron#1276 +METRON-1887: Add logging to the ClasspathFunctionResolver (mmiklavc via mmiklavc) closes apache/metron#1274 +METRON-1873 Update Bootstrap version in Management UI (sardell) closes apache/metron#1267 +METRON-1825 Upgrade bro to 2.5.5 (JonZeolla via nickwallen) closes apache/metron#1237 +METRON-1890 Metron Vagrant should disable audio (ottobackwards) closes apache/metron#1277 +METRON-1874 Create a Parser Debugger (nickwallen) closes apache/metron#1265 +METRON-1880 Use Caffeine for Profiler Caching (nickwallen) closes apache/metron#1270 +METRON-1877 Nested IF ELSE statements can cause parse errors in Stellar (justinleet) closes apache/metron#1268 +METRON-1872 Move rat plugin away from snapshot version (justinleet) closes apache/metron#1264 +METRON-1875 Expose configurable global settings in the Alerts UI (merrimanr) closes apache/metron#1266 +METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242 +METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (cstella via mmiklavc) +METRON-1749 Update Angular to latest release in Management UI (sardell via nickwallen) closes apache/metron#1217 +METRON-1870 Intermittent Stellar REST test failures (merrimanr via nickwallen) closes apache/metron#1263 +METRON-1868 metron-committer-common incorrectly checking REPO_NAME (JonZeolla via jonzeolla) closes apache/metron#1260 +METRON-1740 Improve Palo Alto parser to handle CONFIG and SYSTEM syslog messages (liuy-tnz via nickwallen) closes apache/metron#1171 +METRON-1847 Create reusable script with functions from prepare-commit (ottobackwards) closes apache/metron#1248 +METRON-1850 Stellar REST function (merrimanr) closes apache/metron#1250 +METRON-1858 BasicFireEyeParser check style cleanup and optimization (ottobackwards) closes apache/metron#1255 +METRON-1864 Stellar date format test fails after daylight saving (ottobackwards) closes apache/metron#1258 +METRON-1861 METRON-1861: REST fails to start when LDAP enabled and 'Active Spring profiles' config is empty (anandsubbu via justinleet) closes apache/metron#1256 +METRON-1853: Add shutdown hook to Stellar BaseFunctionResolver (mmiklavc via mmiklavc) closes apache/metron#1251 +METRON-1857 Fix Metaalert Nested Alert Field Name in Index Template (nickwallen) closes apache/metron#1253 +METRON-1855: Make unified enrichment topology the default and deprecate split-join (mmiklavc via mmiklavc) closes apache/metron#1252 +METRON-1790 Unsubscribe from every observable in the pcap panel UI component (ruffle via nickwallen) closes apache/metron#1208 +METRON-1803: Integrate Cypress with Travis (tiborm via mmiklavc) closes apache/metron#1226 +METRON-1844 Allow for LDAP to be used for authentication and roles (justinleet) closes apache/metron#1246 +METRON-1844 Allow for LDAP to be used for authentication and roles (simonellistonball via justinleet) closes apache
[metron] tag apache-metron_0.7.0-release created (now 2263983)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to tag apache-metron_0.7.0-release in repository https://gitbox.apache.org/repos/asf/metron.git. at 2263983 (commit) No new revisions were added by this update.
[metron] tag apache-metron-0.7.0-rc1 created (now 2263983)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to tag apache-metron-0.7.0-rc1 in repository https://gitbox.apache.org/repos/asf/metron.git. at 2263983 (commit) No new revisions were added by this update.
svn commit: r31498 - in /dev/metron/0.7.0-RC1: ./ CHANGES LICENSE NOTICE apache-metron-0.7.0-rc1.tar.gz apache-metron-0.7.0-rc1.tar.gz.asc apache-metron-0.7.0-rc1.tar.gz.sha256 apache-metron-0.7.0-rc1
Author: leet Date: Tue Dec 11 19:35:08 2018 New Revision: 31498 Log: Adding artifacts for metron 0.7.0-RC1 Added: dev/metron/0.7.0-RC1/ dev/metron/0.7.0-RC1/CHANGES dev/metron/0.7.0-RC1/LICENSE dev/metron/0.7.0-RC1/NOTICE dev/metron/0.7.0-RC1/apache-metron-0.7.0-rc1.tar.gz (with props) dev/metron/0.7.0-RC1/apache-metron-0.7.0-rc1.tar.gz.asc dev/metron/0.7.0-RC1/apache-metron-0.7.0-rc1.tar.gz.sha256 dev/metron/0.7.0-RC1/apache-metron-0.7.0-rc1.tar.gz.sha512 Added: dev/metron/0.7.0-RC1/CHANGES == --- dev/metron/0.7.0-RC1/CHANGES (added) +++ dev/metron/0.7.0-RC1/CHANGES Tue Dec 11 19:35:08 2018 @@ -0,0 +1,98 @@ +METRON-1928 Bump Metron version to 0.7.0 for release. (justinleet) closes apache/metron#1293 +METRON-1931 Update dev utilities to support new repo location (rlenferink via justinleet) closes apache/metron#1295 +METRON-1922 Escaping incorrectly handled in current aesh version (justinleet) closes apache/metron#1291 +METRON-1867 Remove `/api/v1/update/replace` endpoint (nickwallen) closes apache/metron#1284 +METRON-1810 Storm Profiler Intermittent Test Failure (nickwallen) closes apache/metron#1289 +METRON-1909 Remove http filter from release utils changelog generation (justinleet) closes apache/metron#1283 +METRON-1869 Unable to Sort an Escalated Meta Alert (nickwallen) closes apache/metron#1280 +METRON-1889: Add any missing timestamp fields to unified enrichment topology (mmiklavc via mmiklavc) closes apache/metron#1286 +METRON-1913 metron-alert UI - Build broken by missing transitive dependency (tiborm via sardell) closes apache/metron#1285 +METRON-1845 Correct Test Data Load in Elasticsearch Integration Tests (nickwallen) closes apache/metron#1247 +METRON-1888 Default Topology Settings in MPack Cause Profiler to Stall (nickwallen) closes apache/metron#1276 +METRON-1887: Add logging to the ClasspathFunctionResolver (mmiklavc via mmiklavc) closes apache/metron#1274 +METRON-1873 Update Bootstrap version in Management UI (sardell) closes apache/metron#1267 +METRON-1825 Upgrade bro to 2.5.5 (JonZeolla via nickwallen) closes apache/metron#1237 +METRON-1890 Metron Vagrant should disable audio (ottobackwards) closes apache/metron#1277 +METRON-1874 Create a Parser Debugger (nickwallen) closes apache/metron#1265 +METRON-1880 Use Caffeine for Profiler Caching (nickwallen) closes apache/metron#1270 +METRON-1877 Nested IF ELSE statements can cause parse errors in Stellar (justinleet) closes apache/metron#1268 +METRON-1872 Move rat plugin away from snapshot version (justinleet) closes apache/metron#1264 +METRON-1875 Expose configurable global settings in the Alerts UI (merrimanr) closes apache/metron#1266 +METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (mmiklavc via mmiklavc) closes apache/metron#1242 +METRON-1834: Migrate Elasticsearch from TransportClient to new Java REST API (cstella via mmiklavc) +METRON-1749 Update Angular to latest release in Management UI (sardell via nickwallen) closes apache/metron#1217 +METRON-1870 Intermittent Stellar REST test failures (merrimanr via nickwallen) closes apache/metron#1263 +METRON-1868 metron-committer-common incorrectly checking REPO_NAME (JonZeolla via jonzeolla) closes apache/metron#1260 +METRON-1740 Improve Palo Alto parser to handle CONFIG and SYSTEM syslog messages (liuy-tnz via nickwallen) closes apache/metron#1171 +METRON-1847 Create reusable script with functions from prepare-commit (ottobackwards) closes apache/metron#1248 +METRON-1850 Stellar REST function (merrimanr) closes apache/metron#1250 +METRON-1858 BasicFireEyeParser check style cleanup and optimization (ottobackwards) closes apache/metron#1255 +METRON-1864 Stellar date format test fails after daylight saving (ottobackwards) closes apache/metron#1258 +METRON-1861 METRON-1861: REST fails to start when LDAP enabled and 'Active Spring profiles' config is empty (anandsubbu via justinleet) closes apache/metron#1256 +METRON-1853: Add shutdown hook to Stellar BaseFunctionResolver (mmiklavc via mmiklavc) closes apache/metron#1251 +METRON-1857 Fix Metaalert Nested Alert Field Name in Index Template (nickwallen) closes apache/metron#1253 +METRON-1855: Make unified enrichment topology the default and deprecate split-join (mmiklavc via mmiklavc) closes apache/metron#1252 +METRON-1790 Unsubscribe from every observable in the pcap panel UI component (ruffle via nickwallen) closes apache/metron#1208 +METRON-1803: Integrate Cypress with Travis (tiborm via mmiklavc) closes apache/metron#1226 +METRON-1844 Allow for LDAP to be used for authentication and roles (justinleet) closes apache/metron#1246 +METRON-1844 Allow for LDAP to be used for authentication and roles (simonellistonball via
[metron] branch Metron_0.7.0 created (now 2263983)
This is an automated email from the ASF dual-hosted git repository. leet pushed a change to branch Metron_0.7.0 in repository https://gitbox.apache.org/repos/asf/metron.git. at 2263983 METRON-1928 Bump Metron version to 0.7.0 for release. (justinleet) closes apache/metron#1293 No new revisions were added by this update.
[metron] branch master updated: METRON-1928 Bump Metron version to 0.7.0 for release. (justinleet) closes apache/metron#1293
This is an automated email from the ASF dual-hosted git repository. leet pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron.git The following commit(s) were added to refs/heads/master by this push: new 2263983 METRON-1928 Bump Metron version to 0.7.0 for release. (justinleet) closes apache/metron#1293 2263983 is described below commit 2263983761e77b7eec52f70f5e8f8001bac83125 Author: justinleet AuthorDate: Tue Dec 11 09:41:40 2018 -0500 METRON-1928 Bump Metron version to 0.7.0 for release. (justinleet) closes apache/metron#1293 --- Upgrading.md | 2 +- metron-analytics/metron-maas-common/pom.xml | 2 +- metron-analytics/metron-maas-service/pom.xml | 2 +- metron-analytics/metron-profiler-client/pom.xml | 2 +- metron-analytics/metron-profiler-common/pom.xml | 2 +- metron-analytics/metron-profiler-repl/pom.xml | 2 +- metron-analytics/metron-profiler-spark/pom.xml| 2 +- metron-analytics/metron-profiler-storm/pom.xml| 2 +- metron-analytics/metron-statistics/pom.xml| 2 +- metron-analytics/pom.xml | 2 +- metron-contrib/metron-docker/pom.xml | 2 +- metron-contrib/metron-performance/pom.xml | 2 +- metron-contrib/pom.xml| 2 +- metron-deployment/amazon-ec2/conf/defaults.yml| 2 +- metron-deployment/ansible/playbooks/docker_probe_install.yml | 2 +- metron-deployment/ansible/roles/ambari_master/defaults/main.yml | 2 +- .../development/centos6/ansible/inventory/group_vars/all | 2 +- .../development/ubuntu14/ansible/inventory/group_vars/all | 2 +- metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml| 4 ++-- metron-deployment/packaging/ambari/metron-mpack/pom.xml | 4 ++-- .../packaging/ambari/metron-mpack/src/main/resources/mpack.json | 2 +- metron-deployment/packaging/docker/deb-docker/pom.xml | 4 ++-- metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec | 8 metron-deployment/packaging/docker/rpm-docker/pom.xml | 4 ++-- metron-deployment/pom.xml | 2 +- metron-interface/metron-alerts/package-lock.json | 2 +- metron-interface/metron-alerts/package.json | 2 +- metron-interface/metron-alerts/pom.xml| 2 +- metron-interface/metron-alerts/scripts/package.json | 2 +- metron-interface/metron-config/package-lock.json | 2 +- metron-interface/metron-config/package.json | 2 +- metron-interface/metron-config/pom.xml| 2 +- metron-interface/metron-config/scripts/package.json | 2 +- metron-interface/metron-rest-client/pom.xml | 2 +- metron-interface/metron-rest/pom.xml | 2 +- metron-interface/metron-rest/src/main/resources/application.yml | 2 +- metron-interface/pom.xml | 2 +- metron-platform/Performance-tuning-guide.md | 4 ++-- metron-platform/README.md | 2 +- metron-platform/elasticsearch-shaded/pom.xml | 2 +- metron-platform/metron-common/pom.xml | 2 +- metron-platform/metron-common/src/main/scripts/cluster_info.py| 2 +- metron-platform/metron-data-management/pom.xml| 2 +- metron-platform/metron-elasticsearch/pom.xml | 2 +- metron-platform/metron-enrichment/pom.xml | 2 +- metron-platform/metron-hbase-client/pom.xml | 2 +- metron-platform/metron-hbase/pom.xml | 2 +- metron-platform/metron-indexing/pom.xml | 2 +- metron-platform/metron-integration-test/pom.xml | 2 +- metron-platform/metron-job/pom.xml| 2 +- metron-platform/metron-management/pom.xml | 2 +- metron-platform/metron-parsers/pom.xml| 2 +- metron-platform/metron-pcap-backend/pom.xml | 2 +- .../metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh | 2 +- metron-platform/metron-pcap/pom.xml | 2 +- metron-platform/metron-solr/pom.xml | 2 +- metron-platform/metron-storm-kafka-override/pom.xml | 2 +- metron-platform/metron-storm-kafka/pom.xml| 2 +- metron-platform/metron-test-utilities/pom.xml | 2 +- metron-platform
[metron] branch master updated: METRON-1931 Update dev utilities to support new repo location (rlenferink via justinleet) closes apache/metron#1295
This is an automated email from the ASF dual-hosted git repository.
leet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new f8776f5 METRON-1931 Update dev utilities to support new repo location
(rlenferink via justinleet) closes apache/metron#1295
f8776f5 is described below
commit f8776f559c7fa9dcbf857ce29636a45b3a38791b
Author: rlenferink
AuthorDate: Mon Dec 10 16:57:13 2018 -0500
METRON-1931 Update dev utilities to support new repo location (rlenferink
via justinleet) closes apache/metron#1295
---
dev-utilities/committer-utils/README.md | 2 +-
dev-utilities/committer-utils/metron-committer-common | 4 ++--
dev-utilities/release-utils/README.md | 2 +-
dev-utilities/release-utils/prepare-release-candidate | 6 +++---
dev-utilities/release-utils/validate-jira-for-release | 4 ++--
metron-analytics/pom.xml | 6 +++---
metron-contrib/pom.xml| 6 +++---
metron-interface/pom.xml | 6 +++---
metron-platform/pom.xml | 6 +++---
metron-stellar/pom.xml| 6 +++---
site-book/pom.xml | 6 +++---
site/BUILD.md | 2 +-
12 files changed, 28 insertions(+), 28 deletions(-)
diff --git a/dev-utilities/committer-utils/README.md
b/dev-utilities/committer-utils/README.md
index ac428cc..0a91759 100644
--- a/dev-utilities/committer-utils/README.md
+++ b/dev-utilities/committer-utils/README.md
@@ -54,7 +54,7 @@ In the following example, I enter the pull request number
when prompted. Using
remote: Total 37861 (delta 27), reused 47 (delta 4), pack-reused 37757
Receiving objects: 100% (37861/37861), 58.18 MiB | 4.38 MiB/s, done.
Resolving deltas: 100% (14439/14439), done.
-From https://git-wip-us.apache.org/repos/asf/metron
+From https://gitbox.apache.org/repos/asf/metron.git
* branch master -> FETCH_HEAD
* [new branch]master -> upstream/master
Already on 'master'
diff --git a/dev-utilities/committer-utils/metron-committer-common
b/dev-utilities/committer-utils/metron-committer-common
index 0f6c06a..23ec4e9 100644
--- a/dev-utilities/committer-utils/metron-committer-common
+++ b/dev-utilities/committer-utils/metron-committer-common
@@ -21,9 +21,9 @@
#
# the upstream apache git repo for apache metron
-METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron.git;
+METRON_UPSTREAM="https://gitbox.apache.org/repos/asf/metron.git;
# the upstream apache git repo for apache metron bro plugin kafka
-BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/metron-bro-plugin-kafka.git;
+BRO_PLUGIN_UPSTREAM="https://gitbox.apache.org/repos/asf/metron-bro-plugin-kafka.git;
# the common configuration file with the committer info
CONFIG_FILE=~/.metron-prepare-commit
diff --git a/dev-utilities/release-utils/README.md
b/dev-utilities/release-utils/README.md
index d1fd816..b7b5cb0 100644
--- a/dev-utilities/release-utils/README.md
+++ b/dev-utilities/release-utils/README.md
@@ -89,7 +89,7 @@ In the following example, enter the appropriate information
Checking out repo: dev
Checking out repo: https://dist.apache.org/repos/dist/release/metron
Checking out repo: release
- Checking out git repo: https://git-wip-us.apache.org/repos/asf/metron.git
+ Checking out git repo: https://gitbox.apache.org/repos/asf/metron.git
Cloning into '/Users/justinleet/tmp/metron-0.6.1/metron'...
remote: Counting objects: 46146, done.
remote: Compressing objects: 100% (15568/15568), done.
diff --git a/dev-utilities/release-utils/prepare-release-candidate
b/dev-utilities/release-utils/prepare-release-candidate
index 35f5013..cb0585d 100755
--- a/dev-utilities/release-utils/prepare-release-candidate
+++ b/dev-utilities/release-utils/prepare-release-candidate
@@ -28,12 +28,12 @@ set -eo pipefail
# if versioning of the submodule isn't x.y.z format, retrieval of the git tag
must also be adjusted.
METRON_REPO_NAME="metron"
BRO_PLUGIN_REPO_NAME="metron-bro-plugin-kafka"
-METRON_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${METRON_REPO_NAME}.git;
-BRO_PLUGIN_UPSTREAM="https://git-wip-us.apache.org/repos/asf/${BRO_PLUGIN_REPO_NAME}.git;
+METRON_UPSTREAM="https://gitbox.apache.org/repos/asf/${METRON_REPO_NAME}.git;
+BRO_PLUGIN_UPSTREAM="https://gitbox.apache.org/repos/asf/${BRO_PLUGIN_REPO_NAME}.git;
DEV_REPO="https://dist.apache.org/repos/dist/dev/metron;
RELEASE_REPO=" https://dist.apache.org/repos/dist/release/metron;
-PLUGIN_GIT_REPO="https://git-wip-us.apache.org/repos/asf/${BRO_PLUGIN_REPO_NAME}.git;
+PLUGIN_GIT_REPO="https://gitbox.apache.org/
metron git commit: METRON-1922 Escaping incorrectly handled in current aesh version (justinleet) closes apache/metron#1291
Repository: metron
Updated Branches:
refs/heads/master 285ba5eb6 -> 30fafe5df
METRON-1922 Escaping incorrectly handled in current aesh version (justinleet)
closes apache/metron#1291
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/30fafe5d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/30fafe5d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/30fafe5d
Branch: refs/heads/master
Commit: 30fafe5df16386a99274b871a291a708963eaae9
Parents: 285ba5e
Author: justinleet
Authored: Mon Dec 10 10:19:30 2018 -0500
Committer: leet
Committed: Mon Dec 10 10:19:30 2018 -0500
--
dependencies_with_url.csv | 3 ++-
metron-platform/metron-common/pom.xml | 2 +-
metron-stellar/stellar-common/pom.xml | 2 +-
.../metron/stellar/common/shell/cli/StellarShellTest.java | 6 ++
pom.xml| 1 +
5 files changed, 11 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/30fafe5d/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index bdc2508..0a63922 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -1,5 +1,5 @@
com.jakewharton.fliptables:fliptables:jar:1.0.2:compile,Apache
v2,https://github.com/JakeWharton/flip-tables
-org.jboss.aesh:aesh:jar:0.66.8:compile,Apache
v2,https://github.com/aeshell/aesh
+org.jboss.aesh:aesh:jar:0.66.19:compile,Apache
v2,https://github.com/aeshell/aesh
org.objenesis:objenesis:jar:1.2:compile,Apache v2,http://objenesis.org/
org.objenesis:objenesis:jar:2.1:compile,Apache v2,http://objenesis.org/
org.ow2.asm:asm:jar:4.1:compile,BSD,http://asm.ow2.org/
@@ -489,3 +489,4 @@
org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https:
com.github.palindromicity:simple-syslog-5424:jar:0.0.9:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
org.elasticsearch.client:elasticsearch-rest-high-level-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
org.elasticsearch.plugin:aggs-matrix-stats-client:jar:5.6.2:compile,ASLv2,https://github.com/elastic/elasticsearch/blob/master/LICENSE.txt
+org.fusesource.jansi:jansi:jar:1.16:compile,ASLv2,https://github.com/fusesource/jansi/blob/master/license.txt
http://git-wip-us.apache.org/repos/asf/metron/blob/30fafe5d/metron-platform/metron-common/pom.xml
--
diff --git a/metron-platform/metron-common/pom.xml
b/metron-platform/metron-common/pom.xml
index 1ca8601..afa4e64 100644
--- a/metron-platform/metron-common/pom.xml
+++ b/metron-platform/metron-common/pom.xml
@@ -353,7 +353,7 @@
org.jboss.aesh
aesh
-0.66.8
+${global_aesh_version}
com.clearspring.analytics
http://git-wip-us.apache.org/repos/asf/metron/blob/30fafe5d/metron-stellar/stellar-common/pom.xml
--
diff --git a/metron-stellar/stellar-common/pom.xml
b/metron-stellar/stellar-common/pom.xml
index b8be521..b3a1fac 100644
--- a/metron-stellar/stellar-common/pom.xml
+++ b/metron-stellar/stellar-common/pom.xml
@@ -146,7 +146,7 @@
org.jboss.aesh
aesh
-0.66.8
+${global_aesh_version}
com.googlecode.json-simple
http://git-wip-us.apache.org/repos/asf/metron/blob/30fafe5d/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java
--
diff --git
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java
index ef2475b..b8f0997 100644
---
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java
+++
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/shell/cli/StellarShellTest.java
@@ -100,6 +100,12 @@ public class StellarShellTest {
assertEquals("4", stdout());
}
+ @Test
+ public void testBackslashInStrings() throws Exception {
+stellarShell.execute(createOp("SPLIT('foobar', '')"));
+assertEquals("[foo, bar]", stdout());
+ }
+
/**
* Ensure that Stellar lists are displayed correctly in the REPL.
*/
http://git-wip-us.apache.org/repos
metron git commit: METRON-1909 Remove http filter from release utils changelog generation (justinleet) closes apache/metron#1283
Repository: metron
Updated Branches:
refs/heads/master 877b51014 -> e81a5c102
METRON-1909 Remove http filter from release utils changelog generation
(justinleet) closes apache/metron#1283
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e81a5c10
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e81a5c10
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e81a5c10
Branch: refs/heads/master
Commit: e81a5c102375d3b7f9692a561760ff8f8526a6ec
Parents: 877b510
Author: justinleet
Authored: Thu Dec 6 14:29:22 2018 -0500
Committer: leet
Committed: Thu Dec 6 14:29:22 2018 -0500
--
dev-utilities/release-utils/prepare-release-candidate | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/e81a5c10/dev-utilities/release-utils/prepare-release-candidate
--
diff --git a/dev-utilities/release-utils/prepare-release-candidate
b/dev-utilities/release-utils/prepare-release-candidate
index 635285c..35f5013 100755
--- a/dev-utilities/release-utils/prepare-release-candidate
+++ b/dev-utilities/release-utils/prepare-release-candidate
@@ -293,7 +293,7 @@ mv "${ARTIFACT}.tar.gz.asc" "$ART_DIR"
# Do this by getting all commits in current branch that aren't in current
release. Filter out any merges by making sure lines start with blankspace
followed by "METRON"
# i.e. make sure the lines starts with a ticket number to avoid merge commits
into feature branches
printf "Creating CHANGES file\n"
-git log "${CAPITAL_REPO}_${VERSION}" "^tags/${TAG}" --no-merges | grep -E
"^[[:blank:]]+METRON" | sed 's/\[//g' | sed 's/\]//g' | grep -v "http" >
"${ART_DIR}/CHANGES"
+git log "${CAPITAL_REPO}_${VERSION}" "^tags/${TAG}" --no-merges | grep -E
"^[[:blank:]]+METRON" | sed 's/\[//g' | sed 's/\]//g' > "${ART_DIR}/CHANGES"
if [[ $? -ne 0 ]]; then
"Error creating CHANGES file"
exit 1
[metron-bro-plugin-kafka] Git Push Summary
Repository: metron-bro-plugin-kafka Updated Tags: refs/tags/apache-metron-bro-plugin-kafka_0.3.0-rc1 [created] 667922f66
svn commit: r31173 - /dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/
Author: leet Date: Tue Nov 27 19:15:14 2018 New Revision: 31173 Log: Adding artifacts for metron-bro-plugin-kafka 0.3.0-RC1 Added: dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/ dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/CHANGES dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/COPYING dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/NOTICE dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/apache-metron-bro-plugin-kafka_0.3.0-rc1.tar.gz (with props) dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/apache-metron-bro-plugin-kafka_0.3.0-rc1.tar.gz.asc dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/apache-metron-bro-plugin-kafka_0.3.0-rc1.tar.gz.sha256 dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/apache-metron-bro-plugin-kafka_0.3.0-rc1.tar.gz.sha512 Added: dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/CHANGES == --- dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/CHANGES (added) +++ dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/CHANGES Tue Nov 27 19:15:14 2018 @@ -0,0 +1,9 @@ +METRON-1885 Remove version from bro plugin btest (JonZeolla) closes apache/metron-bro-plugin-kafka#19 +METRON-1827 Update librdkafka in metron-bro-plugin-kafka (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#13 +METRON-1866 Improve metron-bro-plugin-kafka documentation (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#17 +METRON-1304 Allow metron-bro-plugin-kafka to include or exclude logs (JonZeolla via nickwallen) closes apache/metron-bro-plugin-kafka#2 +METRON-1865 Fix metron-bro-plugin-kafka tests (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#16 +METRON-1828 Improve bro plugin contributing documentation (JonZeolla) closes apache/metron-bro-plugin-kafka#14 +METRON-1818 Remove config_files from bro-pkg.meta (JonZeolla) closes apache/metron-bro-plugin-kafka#11 +METRON-1800 Increment metron-bro-plugin-kafka version (JonZeolla via jonzeolla) closes apache/metron-bro-plugin-kafka#10 +METRON-1773 Bro plugin docs should refer to Apache Metron project (nickwallen) closes apache/metron-bro-plugin-kafka#9 Added: dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/COPYING == --- dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/COPYING (added) +++ dev/metron/metron-bro-plugin-kafka/0.3.0-RC1/COPYING Tue Nov 27 19:15:14 2018 @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 +http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mea
svn commit: r31172 - /dev/metron/metron-bro-plugin-kafka/
Author: leet Date: Tue Nov 27 19:13:18 2018 New Revision: 31172 Log: Adding subdirectory for metron-bro-plugin-kafka releases Added: dev/metron/metron-bro-plugin-kafka/
[metron-bro-plugin-kafka] Git Push Summary
Repository: metron-bro-plugin-kafka Updated Branches: refs/heads/Metron-bro-plugin-kafka_0.3.0 [created] 667922f66
metron git commit: METRON-1877 Nested IF ELSE statements can cause parse errors in Stellar (justinleet) closes apache/metron#1268
Repository: metron
Updated Branches:
refs/heads/master bfa7491f1 -> 05d309f91
METRON-1877 Nested IF ELSE statements can cause parse errors in Stellar
(justinleet) closes apache/metron#1268
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/05d309f9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/05d309f9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/05d309f9
Branch: refs/heads/master
Commit: 05d309f916c0eb45a287a105f44e16e93e3c378e
Parents: bfa7491
Author: justinleet
Authored: Tue Nov 20 08:36:53 2018 -0500
Committer: leet
Committed: Tue Nov 20 08:36:53 2018 -0500
--
.../metron/stellar/common/StellarCompiler.java | 23 +--
.../stellar/dsl/functions/BasicStellarTest.java | 68
2 files changed, 84 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/05d309f9/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
index 8a328a2..ac5412b 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/StellarCompiler.java
@@ -156,20 +156,21 @@ public class StellarCompiler extends StellarBaseListener {
public Object apply(ExpressionState state) {
Deque> instanceDeque = new ArrayDeque<>();
{
-boolean skipElse = false;
+int skipElseCount = 0;
boolean skipMatchClauses = false;
Token token = null;
for (Iterator> it = getTokenDeque().descendingIterator();
it.hasNext(); ) {
token = it.next();
//if we've skipped an else previously, then we need to skip the
deferred tokens associated with the else.
- if (skipElse && token.getUnderlyingType() == ElseExpr.class) {
+ if (skipElseCount > 0 && token.getUnderlyingType() ==
ElseExpr.class) {
while (it.hasNext()) {
token = it.next();
if (token.getUnderlyingType() == EndConditional.class) {
break;
}
}
-skipElse = false;
+// We've completed a single else.
+skipElseCount--;
}
if (skipMatchClauses && (token.getUnderlyingType() ==
MatchClauseEnd.class
|| token.getUnderlyingType() == MatchClauseCheckExpr.class)) {
@@ -219,14 +220,22 @@ public class StellarCompiler extends StellarBaseListener {
//short circuit the if/then/else
instanceDeque.pop();
if((Boolean)curr.getValue()) {
-//choose then
-skipElse = true;
+//choose then. Need to make sure we're keeping track of
nesting.
+skipElseCount++;
} else {
//choose else
+// Need to count in case we see another if-else, to avoid
breaking on wrong else.
+int innerIfCount = 0;
while (it.hasNext()) {
Token t = it.next();
- if (t.getUnderlyingType() == ElseExpr.class) {
-break;
+ if (t.getUnderlyingType() == IfExpr.class) {
+innerIfCount++;
+ } else if (t.getUnderlyingType() == ElseExpr.class) {
+if (innerIfCount == 0) {
+ break;
+} else {
+ innerIfCount--;
+}
}
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/05d309f9/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
--
diff --git
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
index 79f97bc..97cb970 100644
---
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
+++
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
@@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableSet;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Colle
metron git commit: METRON-1872 Move rat plugin away from snapshot version (justinleet) closes apache/metron#1264
Repository: metron Updated Branches: refs/heads/master 3c4c069b6 -> bfa7491f1 METRON-1872 Move rat plugin away from snapshot version (justinleet) closes apache/metron#1264 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bfa7491f Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bfa7491f Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bfa7491f Branch: refs/heads/master Commit: bfa7491f160abc3e01bb1c6f1c3927eb5cec7021 Parents: 3c4c069 Author: justinleet Authored: Mon Nov 19 09:11:29 2018 -0500 Committer: leet Committed: Mon Nov 19 09:11:29 2018 -0500 -- pom.xml | 18 +- 1 file changed, 1 insertion(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/bfa7491f/pom.xml -- diff --git a/pom.xml b/pom.xml index 9a8bd6a..92d0fda 100644 --- a/pom.xml +++ b/pom.xml @@ -71,22 +71,6 @@ - - -apache.snapshots -Apache Development Snapshot Repository - https://repository.apache.org/content/repositories/snapshots/ -default - -true - - -always -true -warn - - - @ApacheMetron UTF-8 @@ -310,7 +294,7 @@ org.apache.rat apache-rat-plugin -0.13-SNAPSHOT +0.13 verify
metron git commit: METRON-1861 METRON-1861: REST fails to start when LDAP enabled and 'Active Spring profiles' config is empty (anandsubbu via justinleet) closes apache/metron#1256
Repository: metron Updated Branches: refs/heads/master 85cd21aa0 -> 7bba8be0c METRON-1861 METRON-1861: REST fails to start when LDAP enabled and 'Active Spring profiles' config is empty (anandsubbu via justinleet) closes apache/metron#1256 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7bba8be0 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7bba8be0 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7bba8be0 Branch: refs/heads/master Commit: 7bba8be0cb77136459c25af697163d76fcf45100 Parents: 85cd21a Author: anandsubbu Authored: Thu Nov 8 09:08:39 2018 -0500 Committer: leet Committed: Thu Nov 8 09:08:39 2018 -0500 -- .../METRON/CURRENT/package/scripts/params/params_linux.py | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/7bba8be0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 75f68fc..060dfe4 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -62,7 +62,10 @@ metron_jvm_flags = config['configurations']['metron-rest-env']['metron_jvm_flags # Construct the profiles as a temp variable first. Only the first time it's set will carry through metron_spring_profiles_temp = config['configurations']['metron-rest-env']['metron_spring_profiles_active'] if config['configurations']['metron-security-env']['metron.ldap.enabled']: -metron_spring_profiles_active = metron_spring_profiles_temp + ',ldap' +if metron_spring_profiles_temp: +metron_spring_profiles_active = metron_spring_profiles_temp + ',ldap' +else: +metron_spring_profiles_active = 'ldap' else: metron_spring_profiles_active = metron_spring_profiles_temp
[1/2] metron git commit: METRON-1844 Allow for LDAP to be used for authentication and roles (simonellistonball via justinleet) closes apache/metron#1246
Repository: metron
Updated Branches:
refs/heads/master eee996302 -> fefbb376f
METRON-1844 Allow for LDAP to be used for authentication and roles
(simonellistonball via justinleet) closes apache/metron#1246
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d0411f63
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d0411f63
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d0411f63
Branch: refs/heads/master
Commit: d0411f63bb2e08f1cca81a25117fd640689bb413
Parents: eee9963
Author: Simon Elliston Ball
Authored: Wed Oct 17 11:46:17 2018 -0400
Committer: justinjleet
Committed: Fri Nov 2 09:23:52 2018 -0400
--
.../CURRENT/configuration/metron-rest-env.xml | 15 +-
.../configuration/metron-security-env.xml | 139 +++
.../common-services/METRON/CURRENT/metainfo.xml | 3 +
.../package/scripts/alerts_ui_commands.py | 1 +
.../package/scripts/management_ui_commands.py | 1 +
.../package/scripts/params/params_linux.py | 14 ++
.../CURRENT/package/scripts/rest_commands.py| 4 +
.../METRON/CURRENT/package/templates/metron.j2 | 14 ++
.../METRON/CURRENT/themes/metron_theme.json | 129 +
metron-interface/metron-rest/pom.xml| 12 ++
.../src/main/config/rest_application.yml| 31 +++--
.../apache/metron/rest/MetronRestConstants.java | 1 +
.../metron/rest/config/WebSecurityConfig.java | 81 +--
.../rest/controller/AlertsUIController.java | 1 +
.../src/main/resources/application-vagrant.yml | 14 ++
.../metron-rest/src/main/scripts/metron-rest.sh | 5 +-
16 files changed, 431 insertions(+), 34 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/d0411f63/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index f4b2327..ab2491b 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -35,32 +35,29 @@
metron_spring_profiles_active
-Active Spring profiles
+Active Spring profiles. 'jdbc' is the default profiler
for authentication. 'ldap' can is also available.
Active Spring profiles
-
-
-true
-
+jdbc
-
+
metron_jdbc_driver
Class name of the JDBC Driver used by Metron
Metron JDBC Driver
-
+
metron_jdbc_url
JDBC Connection URL used by Metron
Metron JDBC URL
-
+
metron_jdbc_username
Metron JDBC Username
Metron JDBC username
-
+
metron_jdbc_password
PASSWORD
http://git-wip-us.apache.org/repos/asf/metron/blob/d0411f63/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml
new file mode 100644
index 000..b96653e
--- /dev/null
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml
@@ -0,0 +1,139 @@
+
+
+
+
+
+metron.ldap.url
+LDAP URL
+ldap://localhost:33369
+LDAP Server URL
+
+ false
+
+
+
+
+metron.ldap.user.dnpattern
+uid={0},ou=people,dc=hadoop,dc=apache,dc=org
+User dn pattern
+LDAP user DN
+
+ false
+
+
+
+
+metron.ldap.user.searchbase
+Group Search Base
+ou=people,dc=hadoop,dc=apache,dc=org
+LDAP group searchbase
+
+ false
+ true
+
+
+
+
+metron.ldap.group.searchbase
+Group Search Base
+ou=groups,dc=hadoop,dc=apache,dc=org
+LDAP group searchbase
+
+ false
+ true
+
+
+
+
+metron.ldap.group.searchfilter
+Group Search Filter
+member={0}
+LDAP
[2/2] metron git commit: METRON-1844 Allow for LDAP to be used for authentication and roles (justinleet) closes apache/metron#1246
METRON-1844 Allow for LDAP to be used for authentication and roles (justinleet)
closes apache/metron#1246
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fefbb376
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fefbb376
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fefbb376
Branch: refs/heads/master
Commit: fefbb376fad0526116ed91035408bfeb72059d9c
Parents: d0411f6
Author: justinjleet
Authored: Tue Oct 23 13:29:32 2018 -0400
Committer: justinjleet
Committed: Fri Nov 2 09:24:08 2018 -0400
--
dependencies_with_url.csv | 2 +
metron-deployment/development/README.md | 34 +++
.../development/knox-demo-ldap.ldif | 101 +++
.../CURRENT/configuration/metron-rest-env.xml | 7 +-
.../configuration/metron-security-env.xml | 57 ++-
.../package/scripts/alerts_ui_commands.py | 1 -
.../package/scripts/management_ui_commands.py | 1 -
.../package/scripts/params/params_linux.py | 12 ++-
.../CURRENT/package/scripts/rest_commands.py| 4 +-
.../METRON/CURRENT/package/templates/metron.j2 | 2 +
.../METRON/CURRENT/themes/metron_theme.json | 30 ++
metron-interface/metron-rest/README.md | 38 +++
.../src/main/config/rest_application.yml| 15 +--
.../metron/rest/config/WebSecurityConfig.java | 40 +++-
.../rest/controller/AlertsUIController.java | 1 -
.../metron/rest/controller/UserController.java | 13 +++
.../src/main/resources/application-vagrant.yml | 3 +-
.../metron-rest/src/main/scripts/metron-rest.sh | 23 +
18 files changed, 337 insertions(+), 47 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/fefbb376/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 2e1eedd..7bc0e9b 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -337,6 +337,8 @@
org.springframework:spring-web:jar:5.0.5.RELEASE:compile,The Apache Software Lic
org.springframework:spring-webmvc:jar:3.2.6.RELEASE:compile,The Apache
Software License, Version 2.0,https://github.com/SpringSource/spring-framework
org.springframework:spring-webmvc:jar:4.3.3.RELEASE:compile,The Apache
Software License, Version
2.0,https://github.com/spring-projects/spring-framework
org.springframework:spring-webmvc:jar:5.0.5.RELEASE:compile,The Apache
Software License, Version
2.0,https://github.com/spring-projects/spring-framework
+org.springframework.ldap:spring-ldap-core:jar:2.3.2.RELEASE:compile,ASLv2,https://spring.io/projects/spring-ldap
+org.springframework.security:spring-security-ldap:jar:5.1.1.RELEASE:compile,ASLv2,https://spring.io/projects/spring-security
org.tukaani:xz:jar:1.0:compile,Public Domain,http://tukaani.org/xz/java.html
org.xerial.snappy:snappy-java:jar:1.0.4.1:compile,The Apache Software License,
Version 2.0,http://code.google.com/p/snappy-java/
org.xerial.snappy:snappy-java:jar:1.1.1.7:compile,The Apache Software License,
Version 2.0,https://github.com/xerial/snappy-java
http://git-wip-us.apache.org/repos/asf/metron/blob/fefbb376/metron-deployment/development/README.md
--
diff --git a/metron-deployment/development/README.md
b/metron-deployment/development/README.md
index 2a04e5f..b86a5c4 100644
--- a/metron-deployment/development/README.md
+++ b/metron-deployment/development/README.md
@@ -27,3 +27,37 @@ This directory contains environments useful for Metron
developers. These enviro
## Vagrant Cachier recommendations
The development boxes are designed to be spun up and destroyed on a regular
basis as part of the development cycle. In order to avoid the overhead of
re-downloading many of the heavy platform dependencies, Vagrant can use the
[vagrant-cachier](http://fgrehm.viewdocs.io/vagrant-cachier/) plugin to store
package caches between builds. If the plugin has been installed to your vagrant
it will be used, and packages will be cached in ~/.vagrant/cache.
+
+## Knox Demo LDAP
+
+The development environment can be set up to authenticate against Knox's demo
LDAP.
+
+A couple notes
+* A custom LDIF file is used to setup users. This is to get the roles and
passwords setup correctly.
+* The demo LDAP uses plaintext passwords with no encryption prefix (e.g.
{SSHA}).
+* You may need or want to shut down any or all of the topologies. This is
optional, but clears some room
+
+To setup this up, start full dev.
+* In Ambari, add the Knox service (Actions -> +Add Service). Accept all
defaults and let it install. The configs that will be set how we need by
default are:
+ * LDAP URL = ldap://localhost:33389
+ * User
metron git commit: METRON-1799: Remove outdated bylaws from site.
Repository: metron
Updated Branches:
refs/heads/asf-site 58d422b9c -> 572e6decf
METRON-1799: Remove outdated bylaws from site.
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/572e6dec
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/572e6dec
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/572e6dec
Branch: refs/heads/asf-site
Commit: 572e6decf4f1787d2fbfb212dee839c864038b8f
Parents: 58d422b
Author: justinjleet
Authored: Thu Oct 11 08:52:29 2018 -0400
Committer: justinjleet
Committed: Thu Oct 11 08:52:29 2018 -0400
--
develop/bylaws.html | 509 -
develop/bylaws/index.html | 509 -
develop/coding.html | 244
develop/coding/index.html | 244
develop/index.html| 324 --
feed.xml | 6 +-
6 files changed, 3 insertions(+), 1833 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/572e6dec/develop/bylaws.html
--
diff --git a/develop/bylaws.html b/develop/bylaws.html
deleted file mode 100644
index 16fdbd9..000
--- a/develop/bylaws.html
+++ /dev/null
@@ -1,509 +0,0 @@
-
-
-http://ogp.me/ns#;
ng-app="hortonWorksApp">@charset
"UTF-8";[ng\:cloak],[ng-cloak],[data-ng-cloak],[x-ng-cloak],.ng-cloak,.x-ng-cloak,.ng-hide:not(.ng-hide-animate){display:none
!important;}ng\:form{display:block;}.ng-animate-shim{visibility:hidden;}.ng-anchor{position:absolute;}
-
-
-http://gmpg.org/xfn/11;>
-Apache Metron Bylaws
-
-
-img.wp-smiley,
-img.emoji {
-display: inline !important;
-border: none !important;
-box-shadow: none !important;
-height: 1em !important;
-width: 1em !important;
-margin: 0 .07em !important;
-vertical-align: -0.1em !important;
-background: none !important;
-padding: 0 !important;
-}
-
-
-
-
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
-(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new
Date();a=s.createElement(o),
-
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
-
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
-
-ga('create', 'UA-78972211-1', 'auto');
-ga('send', 'pageview');
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-window.lsfDataLayer = window.lsfDataLayer || [];
-
-
-
- body {
-font-family: "roboto-condensed", sans-serif;
- }
-
-
-
-
-
-
-
-
-
-
-
-
-
-/* Copyright 2014 Evernote Corporation. All rights reserved. */
-
-.en-markup-crop-options {
-top: 18px !important;
-left: 50% !important;
-margin-left: -100px !important;
-width: 200px !important;
-border: 2px rgba(255, 255, 255, .38) solid !important;
-border-radius: 4px !important;
-}
-
-.en-markup-crop-options div div:first-of-type {
-margin-left: 0px !important;
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-Home
-
-About
-
-Documentation
-
-Community
-
-News
-
-
-
Download
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-Apache Metron Bylaws
-Introduction
-
-This document defines the bylaws under which the Apache Metron project
-operates. It defines the roles and responsibilities of the project,
-who may vote, how voting works, how conflicts are resolved, etc.
-
-Metron is a project of the Apache Software Foundation (ASF) and the
foundation
-holds the trademark on the name âMetronâ and copyright on the combined
-code base. The http://www.apache.org/foundation/faq.html;>Apache
Foundation
-FAQ and
-http://www.apache.org/foundation/how-it-works.html;>How-It-Works
-explain the operation and background of the foundation.
-
-Apache has a http://www.apache.org/foundation/policies/conduct.html;>code of
metron git commit: METRON-1799 Remove outdated bylaws from site. (justinleet) closes apache/metron#1216
Repository: metron Updated Branches: refs/heads/master f153375fa -> 9b6710053 METRON-1799 Remove outdated bylaws from site. (justinleet) closes apache/metron#1216 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9b671005 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9b671005 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9b671005 Branch: refs/heads/master Commit: 9b6710053894d8a39880cb8157a0e603ed542cb7 Parents: f153375 Author: justinleet Authored: Thu Oct 11 08:41:23 2018 -0400 Committer: leet Committed: Thu Oct 11 08:41:23 2018 -0400 -- site/develop/bylaws.md | 281 site/develop/coding.md | 34 -- site/develop/index.md | 61 -- 3 files changed, 376 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/9b671005/site/develop/bylaws.md -- diff --git a/site/develop/bylaws.md b/site/develop/bylaws.md deleted file mode 100644 index a8fc5fb..000 --- a/site/develop/bylaws.md +++ /dev/null @@ -1,281 +0,0 @@ -layout: page -title: Apache Metron Bylaws - -## Introduction - -This document defines the bylaws under which the Apache Metron project -operates. It defines the roles and responsibilities of the project, -who may vote, how voting works, how conflicts are resolved, etc. - -Metron is a project of the Apache Software Foundation (ASF) and the foundation -holds the trademark on the name "Metron" and copyright on the combined -code base. The [Apache Foundation -FAQ](http://www.apache.org/foundation/faq.html) and -[How-It-Works](http://www.apache.org/foundation/how-it-works.html) -explain the operation and background of the foundation. - -Apache has a [code of -conduct](http://www.apache.org/foundation/policies/conduct.html) that -it expects its members to follow. In particular: - -* Be **open and welcoming**. It is important that we grow and - encourage the community of users and developers for our project. - -* Be **collaborative**. Working together on the open mailing lists and - bug database to make decisions helps the project grow. - -* Be **respectful** of others. Everyone is volunteering their time and - efforts to work on this project. Please be respectful of everyone - and their views. - -Metron is typical of Apache projects in that it operates under a set of -principles, known collectively as the "Apache Way". If you are new to -Apache development, please refer to -[this](http://www.apache.org/foundation/how-it-works.html) for more -information on how Apache projects operate. - -## Roles and Responsibilities - -Apache projects define a set of roles with associated rights and -responsibilities. These roles govern what tasks an individual may -perform within the project. The roles are defined in the following -sections. - -### Users - -The most important participants in the project are people who use our -software. The majority of our developers start out as users and guide -their development efforts from the user's perspective. Users -contribute to the Apache projects by providing feedback to developers -in the form of bug reports and feature suggestions. As well, users -participate in the Apache community by helping other users on mailing -lists and user support forums. - -### Contributors - -Contributors include all of the volunteers who donate time, code, -documentation, -or resources to the Metron Project. A contributor that makes sustained, -welcome contributions to the project may be invited to become a -committer, though the exact timing of such invitations depends on many -factors. - -### Committers - -The project's committers are responsible for the project's technical -management. Committers have the right to commit to the project's git -repository. Committers may cast binding votes on any technical -discussion. - -Committer access is by invitation only and must be approved by -consensus approval of the active Project Management Committee (PMC) -members. - -If a committer wishes to leave the project or does not contribute to -the project in any form for six months, the PMC may make them emeritus. -Emeritus committers lose their ability to commit code or cast binding -votes. An emeritus committer may -request reinstatement of commit access from the PMC. Such -reinstatement is subject to consensus approval of active PMC members. - -All Apache committers are required to have a signed [Individual -Contributor License -Agreement](https://www.apache.org/licenses/icla.txt) (ICLA) on file -with the Apache Software Foundation. There is a [Committer -FAQ](http://www.apache.org/dev/committers.html) which provides more -details on the requirements for Committers. - -A committer who makes a
metron git commit: METRON-1769 Script creation of a release candidate (justinleet) closes apache/metron#1188
Repository: metron Updated Branches: refs/heads/master 3d923cde8 -> f153375fa METRON-1769 Script creation of a release candidate (justinleet) closes apache/metron#1188 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f153375f Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f153375f Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f153375f Branch: refs/heads/master Commit: f153375fad65e75a10ce490b4a5f29f7e82f70dd Parents: 3d923cd Author: justinleet Authored: Wed Oct 10 15:41:22 2018 -0400 Committer: leet Committed: Wed Oct 10 15:41:22 2018 -0400 -- dev-utilities/release-utils/README.md | 129 +++ dev-utilities/release-utils/metron-rc-check | 6 +- .../release-utils/prepare-release-candidate | 337 +++ 3 files changed, 469 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/f153375f/dev-utilities/release-utils/README.md -- diff --git a/dev-utilities/release-utils/README.md b/dev-utilities/release-utils/README.md new file mode 100644 index 000..d1fd816 --- /dev/null +++ b/dev-utilities/release-utils/README.md @@ -0,0 +1,129 @@ + + +# Release Tools + +This project contains tools to assist Apache Metron project committers. + +## Prepare Release Candidate + +This script automates the process of creating a release candidate from `apache/metron` or `apache/metron-bro-plugin-kafka`. The script will prompt for various information necessary. Ensure your signing key is setup per [Release Signing](https://www.apache.org/dev/release-signing.html) and [Apache GnuPGP Instructions](https://www.apache.org/dev/openpgp.html#gnupg) + +When prompted the `[value in brackets]` is used by default. To accept the default, simply press `enter`. If you would like to change the default, type it in and hit `enter` when done. + +In the following example, enter the appropriate information + +1. Execute the script. + +The first time the script is run, you will be prompted for additional information including your Apache username and Apache email. These values are persisted in `~/.metron-prepare-release-candidate`. Subsequent executions of the script will retrieve these values, rather than prompting you again for them. + +``` +$ ./prepare-release-candidate + your apache userid []: leet + your apache email [l...@apache.org]: +``` + +1. Select a repository we're creating an RC for. + +``` +[1] metron +[2] metron-bro-plugin-kafka + which repo? [1]: 1 +``` + +1. Enter the current version number. This will be the base for the CHANGES file + +``` + current version: 0.6.0 +``` + +1. Enter the version being built. + +``` + version being built: 0.6.1 +``` + +1. Enter the current RC number + +``` + release candidate number: 1 +``` + +1. Enter the branch we're releasing from. In most cases, this will be master, but for maintenance releases it can be another branch. + +``` + base revision branch or hash for release candidate [master]: +``` + +1. Enter the signing key id. + +``` + signing key id in 8-byte format (e.g. BADDCAFEDEADBEEF): +``` + +1. Enter if this is a practice run. In a practice run, nothing is pushed to SVN, but everything is setup and built otherwise. + +``` + do a live run (push to remote repositories?) [y/n] +``` + +1. Wait for all repos to be checked out to complete. There will be some additional work done, e.g. along with branch and tag creation. In a live run, you may be prompted for Git credentials to push a branch. + +``` + Checking out repo: https://dist.apache.org/repos/dist/dev/metron + Checking out repo: dev + Checking out repo: https://dist.apache.org/repos/dist/release/metron + Checking out repo: release + Checking out git repo: https://git-wip-us.apache.org/repos/asf/metron.git + Cloning into '/Users/justinleet/tmp/metron-0.6.1/metron'... + remote: Counting objects: 46146, done. + remote: Compressing objects: 100% (15568/15568), done. + remote: Total 46146 (delta 21513), reused 43696 (delta 19489) + Receiving objects: 100% (46146/46146), 56.00 MiB | 1.04 MiB/s, done. + Resolving deltas: 100% (21513/21513), done. + Creating branch: Metron_0.6.1 + Using git rev: master + Already on 'master' + Your branch is up to date with 'origin/master'. + Switched to a new branch 'Metron_0.6.1' + This is a practice run. Not running + Creating tentative git tag <0.6.1-rc1>. Do not push this tag until RC is ready for community review. + Already on 'Metron_0.6.1' + Creating the RC tarbal
metron git commit: METRON-1808 Add Ansible created pyc to gitignore (justinleet) closes apache/metron#1228
Repository: metron Updated Branches: refs/heads/master feb9153a6 -> 891bf4952 METRON-1808 Add Ansible created pyc to gitignore (justinleet) closes apache/metron#1228 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/891bf495 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/891bf495 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/891bf495 Branch: refs/heads/master Commit: 891bf4952ab8eceabcb44659e7498d62005b6aa2 Parents: feb9153 Author: justinleet Authored: Fri Oct 5 16:14:20 2018 -0400 Committer: leet Committed: Fri Oct 5 16:14:20 2018 -0400 -- metron-deployment/.gitignore | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/metron/blob/891bf495/metron-deployment/.gitignore -- diff --git a/metron-deployment/.gitignore b/metron-deployment/.gitignore index 6038b6b..f3d4a2a 100644 --- a/metron-deployment/.gitignore +++ b/metron-deployment/.gitignore @@ -1,3 +1,4 @@ keys/ .vagrant *.retry +*.pyc
[1/4] metron git commit: METRON-1780: Fix broken website images
Repository: metron Updated Branches: refs/heads/asf-site 0bea5bdbe -> 58d422b9c http://git-wip-us.apache.org/repos/asf/metron/blob/58d422b9/current-book/metron-platform/metron-indexing/index.html -- diff --git a/current-book/metron-platform/metron-indexing/index.html b/current-book/metron-platform/metron-indexing/index.html index 65c0bad..0057389 100644 --- a/current-book/metron-platform/metron-indexing/index.html +++ b/current-book/metron-platform/metron-indexing/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Indexing @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Indexing -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/58d422b9/current-book/metron-platform/metron-job/index.html -- diff --git a/current-book/metron-platform/metron-job/index.html b/current-book/metron-platform/metron-job/index.html index 388ea80..43a8cc2 100644 --- a/current-book/metron-platform/metron-job/index.html +++ b/current-book/metron-platform/metron-job/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Metron Job @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Metron Job -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/58d422b9/current-book/metron-platform/metron-management/index.html -- diff --git a/current-book/metron-platform/metron-management/index.html b/current-book/metron-platform/metron-management/index.html index 0aa9eda..0c8109c 100644 --- a/current-book/metron-platform/metron-management/index.html +++ b/current-book/metron-platform/metron-management/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Stellar REPL Management Utilities @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Stellar REPL Management Utilities -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/58d422b9/current-book/metron-platform/metron-parsers/3rdPartyParser.html -- diff --git a/current-book/metron-platform/metron-parsers/3rdPartyParser.html b/current-book/metron-platform/metron-parsers/3rdPartyParser.html index 8d75d41..15798ba 100644 --- a/current-book/metron-platform/metron-parsers/3rdPartyParser.html +++ b/current-book/metron-platform/metron-parsers/3rdPartyParser.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Custom Metron Parsers @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Custom Metron Parsers -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/58d422b9/current-book/metron-platform/metron-parsers/ParserChaining.html -- diff --git a/current-book/metron-platform/metron-parsers/ParserChaining.html b/current-book/metron-platform/metron-parsers/ParserChaining.html index a4c9360..aec793f 100644 --- a/current-book/metron-platform/metron-parsers/ParserChaining.html +++ b/current-book/metron-platform/metron-parsers/ParserChaining.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Parser Chaining @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Parser Chaining -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 @@ -121,7 +121,7 @@ limitations under the License. High Level Solution - + At a high level, we continue to maintain the architectural invariant of a 1-1 relationship between logical sensors and storm topologies. Eventually this relationship may become more complex, but at the moment the approach is to construct a routing parser which will have two responsibilities:
[3/4] metron git commit: METRON-1780: Fix broken website images
http://git-wip-us.apache.org/repos/asf/metron/blob/58d422b9/current-book/images/unified_enrichment_arch.svg -- diff --git a/current-book/images/unified_enrichment_arch.svg b/current-book/images/unified_enrichment_arch.svg new file mode 100644 index 000..e42d394 --- /dev/null +++ b/current-book/images/unified_enrichment_arch.svg @@ -0,0 +1,14 @@ + +http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd;> +http://www.w3.org/2000/svg; xmlns:xlink="http://www.w3.org/1999/xlink; width="611px" height="671px" version="1.1" content="mxfile userAgent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36 version=8.3.6 editor=www.draw.io type=devicediagram id=831675e5-f95b-138c-0e1a-018afd21d13b name=Page-17b3Z0qvIkiX8NHn5tzEKuGSeQWKGO+Z5nnn6hm9nnqqsk1XVbXXqtP1l2pa5t4QQBOER7ss9llb8BtPtwU/hUKh9kja/QUBy/AYzv0EQDoH338+B89cBBEd/HcinMvl1CPyXA2Z5pb8fBH4/upZJOv/pxKXvm6Uc/nww7rsujZc/HQunqd//fFrWN3++6xDm6d8dMOOw+fujbpksxR+P9fqX40Ja5sUfdwZfxK9PojCu86lfu9/v9xsEZz9/fn3chn9c6/cHnYsw6fd/dQhmf4Ppqe+XX6/ag06bp2v/6LZf3+P+nU//1u4p7Zb/ky/8bpYtbNb0jxb/tGs5/+iLYmmb+xX4G0z9rbXA/SYJ5yJNfn8TNmXe3a/j+77pdB/Y0mkp7w4lf/+gLZPkueJzjeG5cnvkz9D5X7/MBf3697nsM2SA//XCntfPeIKe63f9Ehe/3+une3/u/NOmZerrv1np7kAqK5uG7pt++mk/jKZEnMDP8b5bfh9qP9f89c2/OjMpp3tUlf3T8LlfnytT v3fU/Vjp8e92Nvg3E94zI+3bdJnO+5Tfv0Cg/+v3/v59WoAI8uv9/i+D7I8pUPzr8QX8fjD8fVznf7v2v9j2fvG7ef/a1K+/MzXbTWVctE/r/63R/7BS2f7MlL+ZUwmjtHn3c/l770T9svTtfULzfED9bfT/0atd36V/MRqWfngGzTz8mrxZeTz2pH7uRv5xFPjjyP06CZfwN5j89Rbihi7/DaJLh9KNHZD5vCfvP5ppF6yd3684/P6LOmjSv/+lr1f3G0Rd98ugARr24xgImYIJjRqRAbfpmiWyDDXwEm1neJ8I+9e8SDErxmQODJIkeexK26ReD7WmfXSWiWJQzt/GJe84+TpdqQtDEmdWiOoUS6NdOMTzYqgR8T27fOOZ9UbTfn3kr6OzyqMy4C093cXpCSyAp/t+dfaaZALEiKR0Fh1zieNaYGZ93y6Dwx6jPhZ+/qNw/P43bp2Gvm7Dch1rle9dqNmr1HahFy5aPmKKqTpOQCYHV656gC2aPhW0g+9vHyeuvgXulIbCdTJlndK1+txmv0c2Fwkf1E7lWL/fhukmLJeFZyF7f/J5R/PTKbeFGs9wqLdLtbLs4QbdE7U6TMkl6acuHBSdshUvhfrTMON1DdcuC1qngfe4pCb7KiQays3HdIxiHUEpBwHcaqwPHHErQuR9Erk9T+vJJ3ikpiCbOBY6DRZSFToRpZ1mzGmc7pTnkulJEYd4qwYkqubD6v2tBLC1dzudxnZfyDHbzpk+2kYZI5/okq8PHoOStMHnMYAxz11Oz9sjUuD5NOkdXzrur3FVgJtL8cog0LnfuTglTI8JanjWKYtDNDZUYBw8S0m4h+EHgkyIninMfuySld59pnl6wkrrWHG2i7nMz4TM+4sndGmWVXxngC22EeDTh+iu3lemtNf1AYmnOVoBXdq6qsL9GlogQV7uOUjZHgOMoNhqQI+93PsAtvCiG1zpB8gVqCde1X3Mo6V7VFBowwYto1Xku4TWD b1Y/CC54UwHOktWcn21pYL26j3PuZSBu+3daQbm9rc9KMlkhbHMaly35ACsJ2OKFka9KgK/Nlxh3gRTKyhLWIXgNkmVhLa79/fX6K3ljh1DGQTpipm/j3AApJvxjKDQ5+6f5deQJbN3iWN371Bq4M8YBrg6rKonR58kxAJvOP0ZE3yd3Gc09uv1/swbaQA8TxSAJAjHkjWFo0V4goufj1l2BW8DysY844VCaABj5ftVsAWEXpPnjLi6V0Tt1DogVHArnxU+kvCDIxTje71P7AbFn+gI3KtNyIdnMqXRBEMDjrB0ksqCdym+TSLnLLsg3Jd1Epqu4Whb3MW8UlKahHZvgckdRWymMV0J/+lPoaWHZ4qIWr6HXiNFmPDxDxOLF9J4uiVTCTg7IP/EmDJUAntMgKS/H4DbDjgfnzk20f7tyblXH6q820Uod6zbVSDaXB80weB29syhLsD4umnMVUrJynjOf0usrqrGpYG5XgIC1rf8WxiQ8BlQW3AC+P6WZ0uFa5wtpV7rI0KRadcEkHheLmfWt7xH5dszcHg6k/5pisQc0HJXVXFENRblazVXfJ5Ouu+eRR3WjQCS0LCmGSGuwJVDewr64tS36IPvtd3H+zTANnF1mj9948Qm4Joju7qR77SFpHloEdLPhOL2GoTfHitWhnhOJEvJsggYDpzWsptDpmwOCLkMsm5nt6enzM8bdYwcjkS9MAZQTyBmczAd8UXE4RVuJ3mv29d+D9K2Ofx4TZXope66td8eTleAushSpuRTY2IjoAva3X+G4jPdKeyA91+uVWEJhjxCDNhV/7RuB0J01Vwt4K9Ph4DACaXfUquPfcDeLToQg+BSldZgrkS6r0dkJsXqgu0LIqm/2lkJQjboda7dxWcKdI+tztttnFE3x1zQZGObX4VLx4lYAKiwPc6g0FSpt5OIuEzw6cgPGJSHWWf+roqYEwf0ceBRxq/3/Tj5cQbv5rlucAxcSxj2SfXsO6CJMX LfgoTr1ek5Hlu7uP/pydGJku1zBweroGZ00d8yxD5+KyGM4KcveKqgWj4Mi/72muDbYRHpjSV5dCJIKjAe+CmtTJoXdRuibJ/GLrdeyeU2mDVb2vg209gG3HaOF3e9u8elshdM8Z04ChCP2xxfMQt75URLuJ3woWz/FYTEExToVCo6dsxtWwl8oZfbT0MrTaQAUpKqUO6GraFfTesBxCfh+tnqYtuBisdfePqV2eg99bks4525C+NK7jfUiiAxm1VUTPTAe1ObUNU11aCvzwDACuOuceKV9Mk7pW6dYH2j2rsLWpdhXiehbEcgFldUuTm5Hn1HzD+hpDu3smqPgwxJHctWhZsS5f1+hx9D74PhMBjbhq9D3xisTfmnOXqJrvWDIkbqczEwjnze8oV8xrCwE+meChRX0fvA1hr3pl3qGX9Ihby6fJCC+KqrQvDR28u6TgBh7+lN3DN1gF5iwIe6XTUYUTmCThtAz8y0jNa8TweY+liQOsasK1I8SOHaHsAEz+/opNTuNIImZTwBoipEBaTHdc8PRX+9ZSSczbBTnOlis4rK/JiZTvc85tdsUQUCacxSadgP/Jjn/JOg9nsrsnkZp9cF84VHf/Kn9y7+QTAc/qonxkVbikBNBFGp+vmQP5UdG5yMZdnG9evhGtGGd0lKR2ZegGz0feMpyVL3wDxx6gYyn5ruFNXeMB3q7+g7gVPDy3R8fGqzC2tDu0SuuYOwfsViv2Sh0155LDXnBXY9jYA7ChRAOq6zNjZghFy1S1noCM7A+34G9Zk1gfDMxpDhVDYmr6vVvOltq9l6eyjL5g4V8bwHSjhejIiWkAe1lcrNfJ64sI0xAwbIYLR9Zr9kuznvmMJVD/T2rPid39flyB25yvTKwa6VwDZO5c8QCBt8+rzDy4F/9/2WLuDrJa3FsU9rb/FaQTIKjiKIZtR2FxOVZ/6MuVZ4r3nXcB/QgswA3XgDRiLquKdkF+xa8DgT1cIqCRZz0Qg
[4/4] metron git commit: METRON-1780: Fix broken website images
METRON-1780: Fix broken website images Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/58d422b9 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/58d422b9 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/58d422b9 Branch: refs/heads/asf-site Commit: 58d422b9c69a269525fdae7d3496342f19d4da93 Parents: 0bea5bd Author: justinjleet Authored: Mon Sep 17 09:21:06 2018 -0400 Committer: justinjleet Committed: Mon Sep 17 09:21:06 2018 -0400 -- current-book/CONTRIBUTING.html | 6 ++--- current-book/Upgrading.html | 6 ++--- .../images/aggregated_parser_chaining_flow.svg | 14 +++ current-book/images/clustered.png | Bin 0 -> 218476 bytes current-book/images/drill_down.png | Bin 0 -> 246210 bytes current-book/images/find_alerts.png | Bin 0 -> 581508 bytes .../images/message_routing_high_level.svg | 14 +++ current-book/images/performance_measurement.png | Bin 0 -> 5790 bytes current-book/images/squid_search.png| Bin 0 -> 161855 bytes current-book/images/unified_enrichment_arch.svg | 14 +++ current-book/index.html | 6 ++--- current-book/metron-analytics/index.html| 6 ++--- .../metron-maas-service/index.html | 6 ++--- .../metron-profiler-client/index.html | 6 ++--- .../metron-analytics/metron-profiler/index.html | 6 ++--- .../metron-statistics/HLLP.html | 6 ++--- .../metron-statistics/index.html| 6 ++--- .../metron-contrib/metron-docker/index.html | 6 ++--- .../metron-performance/index.html | 8 +++--- .../Kerberos-ambari-setup.html | 6 ++--- .../Kerberos-manual-setup.html | 6 ++--- .../metron-deployment/amazon-ec2/index.html | 6 ++--- .../metron-deployment/ansible/index.html| 6 ++--- .../metron-deployment/ansible/roles/index.html | 6 ++--- .../ansible/roles/opentaxii/index.html | 6 ++--- .../ansible/roles/pcap_replay/index.html| 6 ++--- .../ansible/roles/sensor-stubs/index.html | 6 ++--- .../ansible/roles/sensor-test-mode/index.html | 6 ++--- .../development/centos6/index.html | 6 ++--- .../development/fastcapa/index.html | 6 ++--- .../metron-deployment/development/index.html| 6 ++--- .../development/ubuntu14/index.html | 6 ++--- current-book/metron-deployment/index.html | 6 ++--- .../metron-deployment/other-examples/index.html | 6 ++--- .../manual-install/Manual_Install_CentOS6.html | 6 ++--- .../ambari/elasticsearch-mpack/index.html | 6 ++--- .../packaging/ambari/index.html | 6 ++--- .../packaging/ambari/metron-mpack/index.html| 6 ++--- .../packaging/docker/ansible-docker/index.html | 6 ++--- .../packaging/docker/deb-docker/index.html | 6 ++--- .../packaging/docker/rpm-docker/index.html | 6 ++--- .../packaging/packer-build/index.html | 6 ++--- .../metron-interface/metron-alerts/index.html | 25 +-- .../metron-interface/metron-config/index.html | 11 .../metron-interface/metron-rest/index.html | 6 ++--- .../Performance-tuning-guide.html | 6 ++--- current-book/metron-platform/index.html | 6 ++--- .../metron-platform/metron-common/index.html| 6 ++--- .../metron-data-management/index.html | 6 ++--- .../metron-elasticsearch/index.html | 6 ++--- .../metron-enrichment/Performance.html | 6 ++--- .../metron-enrichment/index.html| 8 +++--- .../metron-platform/metron-indexing/index.html | 6 ++--- .../metron-platform/metron-job/index.html | 6 ++--- .../metron-management/index.html| 6 ++--- .../metron-parsers/3rdPartyParser.html | 6 ++--- .../metron-parsers/ParserChaining.html | 8 +++--- .../metron-platform/metron-parsers/index.html | 6 ++--- .../metron-parsers/parser-testing.html | 6 ++--- .../metron-pcap-backend/index.html | 6 ++--- .../metron-platform/metron-solr/index.html | 6 ++--- .../metron-platform/metron-writer/index.html| 6 ++--- current-book/metron-sensors/fastcapa/index.html | 6 ++--- current-book/metron-sensors/index.html | 6 ++--- current-book/metron-sensors/pycapa/index.html | 6 ++--- .../stellar-3rd-party-example/index.html| 6 ++--- .../stellar-common/3rdPartyStellar.html | 6 ++--- .../metron-stellar/stellar-common/index.html| 6 ++--- .../metron-stellar/stellar-zeppelin/index.html | 6 ++--- .../use-cases/forensic_clustering/index.html| 10
[2/4] metron git commit: METRON-1780: Fix broken website images
http://git-wip-us.apache.org/repos/asf/metron/blob/58d422b9/current-book/index.html -- diff --git a/current-book/index.html b/current-book/index.html index 760baf9..f54bd47 100644 --- a/current-book/index.html +++ b/current-book/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Apache Metron @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Apache Metron -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/58d422b9/current-book/metron-analytics/index.html -- diff --git a/current-book/metron-analytics/index.html b/current-book/metron-analytics/index.html index acbd62d..e35ba6d 100644 --- a/current-book/metron-analytics/index.html +++ b/current-book/metron-analytics/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Metron Analytics @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Metron Analytics -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/58d422b9/current-book/metron-analytics/metron-maas-service/index.html -- diff --git a/current-book/metron-analytics/metron-maas-service/index.html b/current-book/metron-analytics/metron-maas-service/index.html index 3f67733..47b0774 100644 --- a/current-book/metron-analytics/metron-maas-service/index.html +++ b/current-book/metron-analytics/metron-maas-service/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Model Management Infrastructure @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Model Management Infrastructure -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/58d422b9/current-book/metron-analytics/metron-profiler-client/index.html -- diff --git a/current-book/metron-analytics/metron-profiler-client/index.html b/current-book/metron-analytics/metron-profiler-client/index.html index b73666c..487ab31 100644 --- a/current-book/metron-analytics/metron-profiler-client/index.html +++ b/current-book/metron-analytics/metron-profiler-client/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Metron Profiler Client @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Metron Profiler Client -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/58d422b9/current-book/metron-analytics/metron-profiler/index.html -- diff --git a/current-book/metron-analytics/metron-profiler/index.html b/current-book/metron-analytics/metron-profiler/index.html index eaa8529..7bf368f 100644 --- a/current-book/metron-analytics/metron-profiler/index.html +++ b/current-book/metron-analytics/metron-profiler/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Metron Profiler @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Metron Profiler -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/58d422b9/current-book/metron-analytics/metron-statistics/HLLP.html -- diff --git a/current-book/metron-analytics/metron-statistics/HLLP.html b/current-book/metron-analytics/metron-statistics/HLLP.html index 09d967c..759f04d 100644 --- a/current-book/metron-analytics/metron-statistics/HLLP.html +++ b/current-book/metron-analytics/metron-statistics/HLLP.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron HyperLogLogPlus @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/
[1/4] metron git commit: METRON-1780 Fix broken website images (justinleet) closes apache/metron#1198
Repository: metron Updated Branches: refs/heads/master 33792c241 -> 02a12806b http://git-wip-us.apache.org/repos/asf/metron/blob/02a12806/site/current-book/metron-platform/metron-indexing/index.html -- diff --git a/site/current-book/metron-platform/metron-indexing/index.html b/site/current-book/metron-platform/metron-indexing/index.html index 65c0bad..0057389 100644 --- a/site/current-book/metron-platform/metron-indexing/index.html +++ b/site/current-book/metron-platform/metron-indexing/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Indexing @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Indexing -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/02a12806/site/current-book/metron-platform/metron-job/index.html -- diff --git a/site/current-book/metron-platform/metron-job/index.html b/site/current-book/metron-platform/metron-job/index.html index 388ea80..43a8cc2 100644 --- a/site/current-book/metron-platform/metron-job/index.html +++ b/site/current-book/metron-platform/metron-job/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Metron Job @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Metron Job -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/02a12806/site/current-book/metron-platform/metron-management/index.html -- diff --git a/site/current-book/metron-platform/metron-management/index.html b/site/current-book/metron-platform/metron-management/index.html index 0aa9eda..0c8109c 100644 --- a/site/current-book/metron-platform/metron-management/index.html +++ b/site/current-book/metron-platform/metron-management/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Stellar REPL Management Utilities @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Stellar REPL Management Utilities -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/02a12806/site/current-book/metron-platform/metron-parsers/3rdPartyParser.html -- diff --git a/site/current-book/metron-platform/metron-parsers/3rdPartyParser.html b/site/current-book/metron-platform/metron-parsers/3rdPartyParser.html index 8d75d41..15798ba 100644 --- a/site/current-book/metron-platform/metron-parsers/3rdPartyParser.html +++ b/site/current-book/metron-platform/metron-parsers/3rdPartyParser.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Custom Metron Parsers @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Custom Metron Parsers -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/02a12806/site/current-book/metron-platform/metron-parsers/ParserChaining.html -- diff --git a/site/current-book/metron-platform/metron-parsers/ParserChaining.html b/site/current-book/metron-platform/metron-parsers/ParserChaining.html index a4c9360..aec793f 100644 --- a/site/current-book/metron-platform/metron-parsers/ParserChaining.html +++ b/site/current-book/metron-platform/metron-parsers/ParserChaining.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Parser Chaining @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Parser Chaining -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 @@ -121,7 +121,7 @@ limitations under the License. High Level Solution - + At a high level, we continue to maintain the architectural invariant of a 1-1 relationship between logical sensors and storm topologies. Eventually this relationship may become more complex, but at the moment the approach
[2/4] metron git commit: METRON-1780 Fix broken website images (justinleet) closes apache/metron#1198
http://git-wip-us.apache.org/repos/asf/metron/blob/02a12806/site/current-book/index.html -- diff --git a/site/current-book/index.html b/site/current-book/index.html index 760baf9..f54bd47 100644 --- a/site/current-book/index.html +++ b/site/current-book/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Apache Metron @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Apache Metron -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/02a12806/site/current-book/metron-analytics/index.html -- diff --git a/site/current-book/metron-analytics/index.html b/site/current-book/metron-analytics/index.html index acbd62d..e35ba6d 100644 --- a/site/current-book/metron-analytics/index.html +++ b/site/current-book/metron-analytics/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Metron Analytics @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Metron Analytics -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/02a12806/site/current-book/metron-analytics/metron-maas-service/index.html -- diff --git a/site/current-book/metron-analytics/metron-maas-service/index.html b/site/current-book/metron-analytics/metron-maas-service/index.html index 3f67733..47b0774 100644 --- a/site/current-book/metron-analytics/metron-maas-service/index.html +++ b/site/current-book/metron-analytics/metron-maas-service/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Model Management Infrastructure @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Model Management Infrastructure -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/02a12806/site/current-book/metron-analytics/metron-profiler-client/index.html -- diff --git a/site/current-book/metron-analytics/metron-profiler-client/index.html b/site/current-book/metron-analytics/metron-profiler-client/index.html index b73666c..487ab31 100644 --- a/site/current-book/metron-analytics/metron-profiler-client/index.html +++ b/site/current-book/metron-analytics/metron-profiler-client/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Metron Profiler Client @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Metron Profiler Client -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/02a12806/site/current-book/metron-analytics/metron-profiler/index.html -- diff --git a/site/current-book/metron-analytics/metron-profiler/index.html b/site/current-book/metron-analytics/metron-profiler/index.html index eaa8529..7bf368f 100644 --- a/site/current-book/metron-analytics/metron-profiler/index.html +++ b/site/current-book/metron-analytics/metron-profiler/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Metron Profiler @@ -32,7 +32,7 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Metron Profiler -| Last Published: 2018-09-12 +| Last Published: 2018-09-13 Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/02a12806/site/current-book/metron-analytics/metron-statistics/HLLP.html -- diff --git a/site/current-book/metron-analytics/metron-statistics/HLLP.html b/site/current-book/metron-analytics/metron-statistics/HLLP.html index 09d967c..759f04d 100644 --- a/site/current-book/metron-analytics/metron-statistics/HLLP.html +++ b/site/current-book/metron-analytics/metron-statistics/HLLP.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> -
[4/4] metron git commit: METRON-1780 Fix broken website images (justinleet) closes apache/metron#1198
METRON-1780 Fix broken website images (justinleet) closes apache/metron#1198 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/02a12806 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/02a12806 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/02a12806 Branch: refs/heads/master Commit: 02a12806b098c5a09a0bc89e925527ae9bdfae1d Parents: 33792c2 Author: justinleet Authored: Mon Sep 17 09:14:16 2018 -0400 Committer: leet Committed: Mon Sep 17 09:14:16 2018 -0400 -- site-book/bin/generate-md.sh| 17 + site/current-book/CONTRIBUTING.html | 6 ++--- site/current-book/Upgrading.html| 6 ++--- .../images/aggregated_parser_chaining_flow.svg | 14 +++ site/current-book/images/clustered.png | Bin 0 -> 218476 bytes site/current-book/images/drill_down.png | Bin 0 -> 246210 bytes site/current-book/images/find_alerts.png| Bin 0 -> 581508 bytes .../images/message_routing_high_level.svg | 14 +++ .../images/performance_measurement.png | Bin 0 -> 5790 bytes site/current-book/images/squid_search.png | Bin 0 -> 161855 bytes .../images/unified_enrichment_arch.svg | 14 +++ site/current-book/index.html| 6 ++--- site/current-book/metron-analytics/index.html | 6 ++--- .../metron-maas-service/index.html | 6 ++--- .../metron-profiler-client/index.html | 6 ++--- .../metron-analytics/metron-profiler/index.html | 6 ++--- .../metron-statistics/HLLP.html | 6 ++--- .../metron-statistics/index.html| 6 ++--- .../metron-contrib/metron-docker/index.html | 6 ++--- .../metron-performance/index.html | 8 +++--- .../Kerberos-ambari-setup.html | 6 ++--- .../Kerberos-manual-setup.html | 6 ++--- .../metron-deployment/amazon-ec2/index.html | 6 ++--- .../metron-deployment/ansible/index.html| 6 ++--- .../metron-deployment/ansible/roles/index.html | 6 ++--- .../ansible/roles/opentaxii/index.html | 6 ++--- .../ansible/roles/pcap_replay/index.html| 6 ++--- .../ansible/roles/sensor-stubs/index.html | 6 ++--- .../ansible/roles/sensor-test-mode/index.html | 6 ++--- .../development/centos6/index.html | 6 ++--- .../development/fastcapa/index.html | 6 ++--- .../metron-deployment/development/index.html| 6 ++--- .../development/ubuntu14/index.html | 6 ++--- site/current-book/metron-deployment/index.html | 6 ++--- .../metron-deployment/other-examples/index.html | 6 ++--- .../manual-install/Manual_Install_CentOS6.html | 6 ++--- .../ambari/elasticsearch-mpack/index.html | 6 ++--- .../packaging/ambari/index.html | 6 ++--- .../packaging/ambari/metron-mpack/index.html| 6 ++--- .../packaging/docker/ansible-docker/index.html | 6 ++--- .../packaging/docker/deb-docker/index.html | 6 ++--- .../packaging/docker/rpm-docker/index.html | 6 ++--- .../packaging/packer-build/index.html | 6 ++--- .../metron-interface/metron-alerts/index.html | 25 +-- .../metron-interface/metron-config/index.html | 11 .../metron-interface/metron-rest/index.html | 6 ++--- .../Performance-tuning-guide.html | 6 ++--- site/current-book/metron-platform/index.html| 6 ++--- .../metron-platform/metron-common/index.html| 6 ++--- .../metron-data-management/index.html | 6 ++--- .../metron-elasticsearch/index.html | 6 ++--- .../metron-enrichment/Performance.html | 6 ++--- .../metron-enrichment/index.html| 8 +++--- .../metron-platform/metron-indexing/index.html | 6 ++--- .../metron-platform/metron-job/index.html | 6 ++--- .../metron-management/index.html| 6 ++--- .../metron-parsers/3rdPartyParser.html | 6 ++--- .../metron-parsers/ParserChaining.html | 8 +++--- .../metron-platform/metron-parsers/index.html | 6 ++--- .../metron-parsers/parser-testing.html | 6 ++--- .../metron-pcap-backend/index.html | 6 ++--- .../metron-platform/metron-solr/index.html | 6 ++--- .../metron-platform/metron-writer/index.html| 6 ++--- .../metron-sensors/fastcapa/index.html | 6 ++--- site/current-book/metron-sensors/index.html | 6 ++--- .../metron-sensors/pycapa/index.html| 6 ++--- .../stellar-3rd-party-example/index.html| 6 ++--- .../stellar-common/3rdPartyStellar.html | 6 ++--- .../metron-stellar/stellar-common/index.html| 6 ++--- .../metron-stellar/stellar-zeppelin/in
[3/4] metron git commit: METRON-1780 Fix broken website images (justinleet) closes apache/metron#1198
http://git-wip-us.apache.org/repos/asf/metron/blob/02a12806/site/current-book/images/unified_enrichment_arch.svg -- diff --git a/site/current-book/images/unified_enrichment_arch.svg b/site/current-book/images/unified_enrichment_arch.svg new file mode 100644 index 000..e42d394 --- /dev/null +++ b/site/current-book/images/unified_enrichment_arch.svg @@ -0,0 +1,14 @@ + +http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd;> +http://www.w3.org/2000/svg; xmlns:xlink="http://www.w3.org/1999/xlink; width="611px" height="671px" version="1.1" content="mxfile userAgent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36 version=8.3.6 editor=www.draw.io type=devicediagram id=831675e5-f95b-138c-0e1a-018afd21d13b name=Page-17b3Z0qvIkiX8NHn5tzEKuGSeQWKGO+Z5nnn6hm9nnqqsk1XVbXXqtP1l2pa5t4QQBOER7ss9llb8BtPtwU/hUKh9kja/QUBy/AYzv0EQDoH338+B89cBBEd/HcinMvl1CPyXA2Z5pb8fBH4/upZJOv/pxKXvm6Uc/nww7rsujZc/HQunqd//fFrWN3++6xDm6d8dMOOw+fujbpksxR+P9fqX40Ja5sUfdwZfxK9PojCu86lfu9/v9xsEZz9/fn3chn9c6/cHnYsw6fd/dQhmf4Ppqe+XX6/ag06bp2v/6LZf3+P+nU//1u4p7Zb/ky/8bpYtbNb0jxb/tGs5/+iLYmmb+xX4G0z9rbXA/SYJ5yJNfn8TNmXe3a/j+77pdB/Y0mkp7w4lf/+gLZPkueJzjeG5cnvkz9D5X7/MBf3697nsM2SA//XCntfPeIKe63f9Ehe/3+une3/u/NOmZerrv1np7kAqK5uG7pt++mk/jKZEnMDP8b5bfh9qP9f89c2/OjMpp3tUlf3T8LlfnytT v3fU/Vjp8e92Nvg3E94zI+3bdJnO+5Tfv0Cg/+v3/v59WoAI8uv9/i+D7I8pUPzr8QX8fjD8fVznf7v2v9j2fvG7ef/a1K+/MzXbTWVctE/r/63R/7BS2f7MlL+ZUwmjtHn3c/l770T9svTtfULzfED9bfT/0atd36V/MRqWfngGzTz8mrxZeTz2pH7uRv5xFPjjyP06CZfwN5j89Rbihi7/DaJLh9KNHZD5vCfvP5ppF6yd3684/P6LOmjSv/+lr1f3G0Rd98ugARr24xgImYIJjRqRAbfpmiWyDDXwEm1neJ8I+9e8SDErxmQODJIkeexK26ReD7WmfXSWiWJQzt/GJe84+TpdqQtDEmdWiOoUS6NdOMTzYqgR8T27fOOZ9UbTfn3kr6OzyqMy4C093cXpCSyAp/t+dfaaZALEiKR0Fh1zieNaYGZ93y6Dwx6jPhZ+/qNw/P43bp2Gvm7Dch1rle9dqNmr1HahFy5aPmKKqTpOQCYHV656gC2aPhW0g+9vHyeuvgXulIbCdTJlndK1+txmv0c2Fwkf1E7lWL/fhukmLJeFZyF7f/J5R/PTKbeFGs9wqLdLtbLs4QbdE7U6TMkl6acuHBSdshUvhfrTMON1DdcuC1qngfe4pCb7KiQays3HdIxiHUEpBwHcaqwPHHErQuR9Erk9T+vJJ3ikpiCbOBY6DRZSFToRpZ1mzGmc7pTnkulJEYd4qwYkqubD6v2tBLC1dzudxnZfyDHbzpk+2kYZI5/okq8PHoOStMHnMYAxz11Oz9sjUuD5NOkdXzrur3FVgJtL8cog0LnfuTglTI8JanjWKYtDNDZUYBw8S0m4h+EHgkyIninMfuySld59pnl6wkrrWHG2i7nMz4TM+4sndGmWVXxngC22EeDTh+iu3lemtNf1AYmnOVoBXdq6qsL9GlogQV7uOUjZHgOMoNhqQI+93PsAtvCiG1zpB8gVqCde1X3Mo6V7VFBowwYto1Xku4TWD b1Y/CC54UwHOktWcn21pYL26j3PuZSBu+3daQbm9rc9KMlkhbHMaly35ACsJ2OKFka9KgK/Nlxh3gRTKyhLWIXgNkmVhLa79/fX6K3ljh1DGQTpipm/j3AApJvxjKDQ5+6f5deQJbN3iWN371Bq4M8YBrg6rKonR58kxAJvOP0ZE3yd3Gc09uv1/swbaQA8TxSAJAjHkjWFo0V4goufj1l2BW8DysY844VCaABj5ftVsAWEXpPnjLi6V0Tt1DogVHArnxU+kvCDIxTje71P7AbFn+gI3KtNyIdnMqXRBEMDjrB0ksqCdym+TSLnLLsg3Jd1Epqu4Whb3MW8UlKahHZvgckdRWymMV0J/+lPoaWHZ4qIWr6HXiNFmPDxDxOLF9J4uiVTCTg7IP/EmDJUAntMgKS/H4DbDjgfnzk20f7tyblXH6q820Uod6zbVSDaXB80weB29syhLsD4umnMVUrJynjOf0usrqrGpYG5XgIC1rf8WxiQ8BlQW3AC+P6WZ0uFa5wtpV7rI0KRadcEkHheLmfWt7xH5dszcHg6k/5pisQc0HJXVXFENRblazVXfJ5Ouu+eRR3WjQCS0LCmGSGuwJVDewr64tS36IPvtd3H+zTANnF1mj9948Qm4Joju7qR77SFpHloEdLPhOL2GoTfHitWhnhOJEvJsggYDpzWsptDpmwOCLkMsm5nt6enzM8bdYwcjkS9MAZQTyBmczAd8UXE4RVuJ3mv29d+D9K2Ofx4TZXope66td8eTleAushSpuRTY2IjoAva3X+G4jPdKeyA91+uVWEJhjxCDNhV/7RuB0J01Vwt4K9Ph4DACaXfUquPfcDeLToQg+BSldZgrkS6r0dkJsXqgu0LIqm/2lkJQjboda7dxWcKdI+tztttnFE3x1zQZGObX4VLx4lYAKiwPc6g0FSpt5OIuEzw6cgPGJSHWWf+roqYEwf0ceBRxq/3/Tj5cQbv5rlucAxcSxj2SfXsO6CJMX LfgoTr1ek5Hlu7uP/pydGJku1zBweroGZ00d8yxD5+KyGM4KcveKqgWj4Mi/72muDbYRHpjSV5dCJIKjAe+CmtTJoXdRuibJ/GLrdeyeU2mDVb2vg209gG3HaOF3e9u8elshdM8Z04ChCP2xxfMQt75URLuJ3woWz/FYTEExToVCo6dsxtWwl8oZfbT0MrTaQAUpKqUO6GraFfTesBxCfh+tnqYtuBisdfePqV2eg99bks4525C+NK7jfUiiAxm1VUTPTAe1ObUNU11aCvzwDACuOuceKV9Mk7pW6dYH2j2rsLWpdhXiehbEcgFldUuTm5Hn1HzD+hpDu3smqPgwxJHctWhZsS5f1+hx9D74PhMBjbhq9D3xisTfmnOXqJrvWDIkbqczEwjnze8oV8xrCwE+meChRX0fvA1hr3pl3qGX9Ihby6fJCC+KqrQvDR28u6TgBh7+lN3DN1gF5iwIe6XTUYUTmCThtAz8y0jNa8TweY+liQOsasK1I8SOHaHsAEz+/opNTuNIImZTwBoipEBaTHdc8PRX+9ZSSczbBTnOlis4rK/JiZTvc85tdsUQUCacxSadgP/Jjn/JOg9nsrsnkZp9cF84VHf/Kn9y7+QTAc/qonxkVbikBNBFGp+vmQP5UdG5yMZdnG9evhGtGGd0lKR2ZegGz0feMpyVL3wDxx6gYyn5ruFNXeMB3q7+g7gVPDy3R8fGqzC2tDu0SuuYOwfsViv2Sh0155LDXnBXY9jYA7ChRAOq6zNjZghFy1S1noCM7A+34G9Zk1gfDMxpDhVDYmr6vVvOltq9l6eyjL5g4V8bwHSjhejIiWkAe1lcrNfJ64sI0xAwbIYLR9Zr9kuznvmMJVD/T2rPid39flyB25yvTKwa6VwDZO5c8QCBt8+rzDy4F/9/2WLuDrJa3FsU9rb/FaQTIKjiKIZtR2FxOVZ/6MuVZ4r3nXcB/QgswA3XgDRiLquKdkF+xa8DgT1cIqCRZz0Qg
svn commit: r29369 - /release/metron/0.5.0/
Author: leet Date: Thu Sep 13 20:57:18 2018 New Revision: 29369 Log: Removing old 0.5.0 release Removed: release/metron/0.5.0/
[4/5] metron git commit: METRON-1776: Update public web site to point at 0.6.0 new release
http://git-wip-us.apache.org/repos/asf/metron/blob/0bea5bdb/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html -- diff --git a/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html b/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html index 5cf9775..f3d8dff 100644 --- a/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html +++ b/current-book/metron-deployment/other-examples/manual-install/Manual_Install_CentOS6.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron @@ -32,8 +32,8 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ -| Last Published: 2018-06-07 - Version: 0.5.0 +| Last Published: 2018-09-12 + Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/0bea5bdb/current-book/metron-deployment/packaging/ambari/elasticsearch-mpack/index.html -- diff --git a/current-book/metron-deployment/packaging/ambari/elasticsearch-mpack/index.html b/current-book/metron-deployment/packaging/ambari/elasticsearch-mpack/index.html index 499e89b..8a59770 100644 --- a/current-book/metron-deployment/packaging/ambari/elasticsearch-mpack/index.html +++ b/current-book/metron-deployment/packaging/ambari/elasticsearch-mpack/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron @@ -32,8 +32,8 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ -| Last Published: 2018-06-07 - Version: 0.5.0 +| Last Published: 2018-09-12 + Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/0bea5bdb/current-book/metron-deployment/packaging/ambari/index.html -- diff --git a/current-book/metron-deployment/packaging/ambari/index.html b/current-book/metron-deployment/packaging/ambari/index.html index bac5758..7acd537 100644 --- a/current-book/metron-deployment/packaging/ambari/index.html +++ b/current-book/metron-deployment/packaging/ambari/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Ambari Management Pack Development @@ -32,8 +32,8 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Ambari Management Pack Development -| Last Published: 2018-06-07 - Version: 0.5.0 +| Last Published: 2018-09-12 + Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/0bea5bdb/current-book/metron-deployment/packaging/ambari/metron-mpack/index.html -- diff --git a/current-book/metron-deployment/packaging/ambari/metron-mpack/index.html b/current-book/metron-deployment/packaging/ambari/metron-mpack/index.html index dd97717..ddf0663 100644 --- a/current-book/metron-deployment/packaging/ambari/metron-mpack/index.html +++ b/current-book/metron-deployment/packaging/ambari/metron-mpack/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron @@ -32,8 +32,8 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ -| Last Published: 2018-06-07 - Version: 0.5.0 +| Last Published: 2018-09-12 + Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/0bea5bdb/current-book/metron-deployment/packaging/docker/ansible-docker/index.html -- diff --git a/current-book/metron-deployment/packaging/docker/ansible-docker/index.html b/current-book/metron-deployment/packaging/docker/ansible-docker/index.html index 8145c8f..4c85d3f 100644 --- a/current-book/metron-deployment/packaging/docker/ansible-docker/index.html +++ b/current-book/metron-deployment/packaging/docker/ansible-docker/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron @@ -32,8 +32,8 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ -| Last Published: 2018-06-07 - Version: 0.5.0 +| Last Published: 2018-09-12 + Version: 0.6.0
[2/5] metron git commit: METRON-1776: Update public web site to point at 0.6.0 new release
http://git-wip-us.apache.org/repos/asf/metron/blob/0bea5bdb/current-book/metron-platform/metron-solr/index.html
--
diff --git a/current-book/metron-platform/metron-solr/index.html
b/current-book/metron-platform/metron-solr/index.html
new file mode 100644
index 000..1afb245
--- /dev/null
+++ b/current-book/metron-platform/metron-solr/index.html
@@ -0,0 +1,300 @@
+
+
+http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
+
+
+
+
+
+Metron Solr in Metron
+
+
+
+
+
+ $( document ).ready( function() { $( '.carousel' ).carousel( {
interval: 3500 } ) } );
+
+
+
+
+
+http://metron.apache.org/;
id="bannerLeft">
+
+
+
+
+
+
+ http://www.apache.org; class="externalLink"
title="Apache">Apache/
+ http://metron.apache.org/; class="externalLink"
title="Metron">Metron/
+ Documentation/
+Solr in Metron
+|
Last Published: 2018-09-12
+ Version: 0.6.0
+
+
+
+
+
+
+ User Documentation
+Metron
+
+CONTRIBUTING
+Upgrading
+Analytics
+Docker
+Performance
+Deployment
+Alerts
+Config
+Rest
+Platform
+
+Performance-tuning-guide
+Common
+Data-management
+Elasticsearch
+Enrichment
+Indexing
+Job
+Management
+Parsers
+Pcap-backend
+Solr
+Writer
+
+
+Sensors
+Stellar-3rd-party-example
+Stellar-common
+Stellar-zeppelin
+Use-cases
+
+
+
+
+
+
+
+
+
+http://maven.apache.org/; title="Built by Maven"
class="poweredBy">
+
+
+
+
+
+Solr in Metron
+
+
+Table of Contents
+
+
+Introduction
+Configuration
+Installing
+Schemas
+Collections
+
+
+Introduction
+Metron ships with Solr 6.6.2 support. Solr Cloud can be used as the
real-time portion of the datastore resulting from metron-indexing.
+
+Configuration
+
+The Indexing Topology
+Solr is a viable option for the random access topology and,
similar to the Elasticsearch Writer, can be configured via the global config.
The following settings are possible as part of the global config:
+
+
+solr.zookeeper
+
+
+The zookeeper quorum associated with the SolrCloud instance. This is a
required field with no default.
+
+
+solr.commitPerBatch
+
+
+This is a boolean which defines whether the writer commits every batch.
The default is true.
+WARNING: If you set this to false, then commits will
happen based on the SolrClients internal mechanism and worker failure
may result data being acknowledged in storm but not written in Solr.
+
+
+solr.commit.soft
+
+
+This is a boolean which defines whether the writer makes a soft commit or
a durable commit. See https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits;>here
The default is false.
+WARNING: If you set this to true, then commits will happen
based on the SolrClients internal mechanism and worker failure
may result data being acknowledged in storm but not written in Solr.
+
+
+solr.commit.waitSearcher
+
+
+This is a boolean which defines whether the writer blocks the commit until
the data is available to search. See https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits;>here
The default is true.
+WARNING: If you set this to false, then commits will
happen based on the SolrClients internal mechanism and worker failure
may result data being acknowledged in storm but not written in Solr.
+
+
+solr.commit.waitFlush
+
+
+This is a boolean which defines whether the writer blocks the commit until
the data is flushed. See https://lucene.apache.org/solr/guide/6_6/near-real-time-searching.html#NearRealTimeSearching-AutoCommits;>here
The default is true.
+WARNING: If you set this to false, then commits will
happen based on the SolrClients internal mechanism and worker failure
may result data being acknowledged in storm but not written in Solr.
+
+
+solr.collection
+
+
+The default solr collection (if unspecified, the name is metron).
By default, sensors will write to a collection associated with the index name
in the indexing config for that sensor. If that index name is the empty
string, then the default collection will be used.
+
+
+solr.http.config
+
+
+This is a map which allows users to configure the Solr clients
HTTP client.
+Possible fields here are:
+
+
+socketTimeout : Socket timeout measured in ms, closes a socket if
read takes longer than x ms to complete throws
java.net.SocketTimeoutException: Read timed out exception
+connTimeout : Connection timeout measures in ms, closes a socket
if connection cannot be established within x ms with a
java.net.SocketTimeoutException: Connection timed out
+maxConectionsPerHost :
[1/5] metron git commit: METRON-1776: Update public web site to point at 0.6.0 new release
Repository: metron
Updated Branches:
refs/heads/asf-site 1a131d88b -> 0bea5bdbe
http://git-wip-us.apache.org/repos/asf/metron/blob/0bea5bdb/current-book/use-cases/parser_chaining/index.html
--
diff --git a/current-book/use-cases/parser_chaining/index.html
b/current-book/use-cases/parser_chaining/index.html
new file mode 100644
index 000..5ddcde2
--- /dev/null
+++ b/current-book/use-cases/parser_chaining/index.html
@@ -0,0 +1,412 @@
+
+
+http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
+
+
+
+
+
+Metron Problem Statement
+
+
+
+
+
+ $( document ).ready( function() { $( '.carousel' ).carousel( {
interval: 3500 } ) } );
+
+
+
+
+
+http://metron.apache.org/;
id="bannerLeft">
+
+
+
+
+
+
+ http://www.apache.org; class="externalLink"
title="Apache">Apache/
+ http://metron.apache.org/; class="externalLink"
title="Metron">Metron/
+ Documentation/
+Problem Statement
+|
Last Published: 2018-09-12
+ Version: 0.6.0
+
+
+
+
+
+
+ User Documentation
+Metron
+
+CONTRIBUTING
+Upgrading
+Analytics
+Docker
+Performance
+Deployment
+Alerts
+Config
+Rest
+Platform
+Sensors
+Stellar-3rd-party-example
+Stellar-common
+Stellar-zeppelin
+Use-cases
+
+Forensic_clustering
+Geographic_login_outliers
+Parser_chaining
+Typosquat_detection
+
+
+
+
+
+
+
+
+
+
+
+http://maven.apache.org/; title="Built by Maven"
class="poweredBy">
+
+
+
+
+
+Problem Statement
+
+Aggregating many different types sensors into a single data source (e.g.
syslog) and ingesting that aggregate sensor into Metron is a common pattern.
It is not obvious precisely how to manage these types of aggregate sensors as
they require two-pass parsing. This document will walk through an example of
supporting this kind of multi-pass ingest.
+Multi-pass parser involves the following requirements:
+
+
+The enveloping parser (e.g. the aggregation format such as syslog or plain
CSV) may contain metadata which should be ingested along with the data.
+The enveloping sensor contains many different sensor types
+
+
+High Level Solution
+
+At a high level, we continue to maintain the architectural invariant of a
1-1 relationship between logical sensors and storm topologies. Eventually this
relationship may become more complex, but at the moment the approach is to
construct a routing parser which will have two responsibilities:
+
+
+Parse the envelope (e.g. syslog data) and extract any metadata fields from
the envelope to pass along
+Route the unfolded data to the appropriate kafka topic associated with the
enveloped sensor data
+
+Because the data emitted from the routing parser is just like any data
emitted from any other parser, in that it is a JSON blob like any data emitted
from any parser, we will need to adjust the downstream parsers to extract the
enveloped data from the JSON blob and treat it as the data to parse.
+
+Example
+
+Preliminaries
+We assume that the following environment variables are set:
+
+
+METRON_HOME - the home directory for metron
+ZOOKEEPER - The zookeeper quorum (comma separated with port
specified: e.g. node1:2181 for full-dev)
+BROKERLIST - The Kafka broker list (comma separated with port
specified: e.g. node1:6667 for full-dev)
+ES_HOST - The elasticsearch master (and port) e.g.
node1:9200 for full-dev.
+
+Before editing configurations, be sure to pull the configs from zookeeper
locally via
+
+
+
+$METRON_HOME/bin/zk_load_configs.sh --mode PULL -z
$ZOOKEEPER -o $METRON_HOME/config/zookeeper/ -f
+
+
+
+The Scenario
+Consider the following situation, we have some logs from a Cisco PIX device
that we would like to ingest. The format is syslog, but multiple scenarios
exist in the same log file. Specificaly, lets consider the sample logs
http://www.monitorware.com/en/logsamples/cisco-pix-61(2).php">here.
+The log lines in general have the following components:
+
+
+A timestamp
+A message type tag
+The message payload that is dependent upon the tag
+
+Lets consider two types of messages that wed like to
parse:
+
+
+Tag 6-302* which are connection creation and teardown messages
e.g. Built UDP connection for faddr 198.207.223.240/53337 gaddr
10.0.0.187/53 laddr 192.168.0.2/53
+Tag 5-304* which are URL access events e.g. 192.168.0.2
Accessed URL 66.102.9.99:/
+
+A couple things are apparent from this:
+
+
+The formats we care about are easy to represent in grok, but are very
different and logically represent very different sensors.
+The syslog loglines output by this device has many types of events that I
do not care about (yet).
+
+We will
[3/5] metron git commit: METRON-1776: Update public web site to point at 0.6.0 new release
http://git-wip-us.apache.org/repos/asf/metron/blob/0bea5bdb/current-book/metron-platform/metron-job/index.html
--
diff --git a/current-book/metron-platform/metron-job/index.html
b/current-book/metron-platform/metron-job/index.html
new file mode 100644
index 000..388ea80
--- /dev/null
+++ b/current-book/metron-platform/metron-job/index.html
@@ -0,0 +1,126 @@
+
+
+http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
+
+
+
+
+
+Metron Metron Job
+
+
+
+
+
+ $( document ).ready( function() { $( '.carousel' ).carousel( {
interval: 3500 } ) } );
+
+
+
+
+
+http://metron.apache.org/;
id="bannerLeft">
+
+
+
+
+
+
+ http://www.apache.org; class="externalLink"
title="Apache">Apache/
+ http://metron.apache.org/; class="externalLink"
title="Metron">Metron/
+ Documentation/
+Metron Job
+|
Last Published: 2018-09-12
+ Version: 0.6.0
+
+
+
+
+
+
+ User Documentation
+Metron
+
+CONTRIBUTING
+Upgrading
+Analytics
+Docker
+Performance
+Deployment
+Alerts
+Config
+Rest
+Platform
+
+Performance-tuning-guide
+Common
+Data-management
+Elasticsearch
+Enrichment
+Indexing
+Job
+Management
+Parsers
+Pcap-backend
+Solr
+Writer
+
+
+Sensors
+Stellar-3rd-party-example
+Stellar-common
+Stellar-zeppelin
+Use-cases
+
+
+
+
+
+
+
+
+
+http://maven.apache.org/; title="Built by Maven"
class="poweredBy">
+
+
+
+
+
+Metron Job
+
+This module holds abstractions for creating jobs. The main actors are a
JobManager interface and subsequent implementation, InMemoryJobManger, that
handles maintaining a cache of running and completed Statusable jobs. Each
Statusable can provide a Finalizer implementation that should be executed on
completion of the underlying job. Successful jobs should return a Pageable
object that allow consumers to request results on a per-page basis.
+
+Job State Statechart
+
+
+
+
+
+
+
+
+é 2015-2016 The Apache Software Foundation. Apache Metron, Metron, Apache,
the Apache feather logo,
+and the Apache Metron project logo are trademarks of The Apache
Software Foundation.
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/metron/blob/0bea5bdb/current-book/metron-platform/metron-management/index.html
--
diff --git a/current-book/metron-platform/metron-management/index.html
b/current-book/metron-platform/metron-management/index.html
index f9ea1ce..0aa9eda 100644
--- a/current-book/metron-platform/metron-management/index.html
+++ b/current-book/metron-platform/metron-management/index.html
@@ -1,13 +1,13 @@
http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
-
+
Metron Stellar REPL Management Utilities
@@ -32,8 +32,8 @@
http://metron.apache.org/; class="externalLink"
title="Metron">Metron/
Documentation/
Stellar REPL Management Utilities
-|
Last Published: 2018-06-07
- Version: 0.5.0
+|
Last Published: 2018-09-12
+ Version: 0.6.0
@@ -55,15 +55,16 @@
Platform
Performance-tuning-guide
-Api
Common
Data-management
Elasticsearch
Enrichment
Indexing
+Job
Management
Parsers
Pcap-backend
+Solr
Writer
http://git-wip-us.apache.org/repos/asf/metron/blob/0bea5bdb/current-book/metron-platform/metron-parsers/3rdPartyParser.html
--
diff --git a/current-book/metron-platform/metron-parsers/3rdPartyParser.html
b/current-book/metron-platform/metron-parsers/3rdPartyParser.html
index 988580b..8d75d41 100644
--- a/current-book/metron-platform/metron-parsers/3rdPartyParser.html
+++ b/current-book/metron-platform/metron-parsers/3rdPartyParser.html
@@ -1,13 +1,13 @@
http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
-
+
Metron Custom Metron Parsers
@@ -32,8 +32,8 @@
http://metron.apache.org/; class="externalLink"
title="Metron">Metron/
Documentation/
Custom Metron Parsers
-|
Last Published: 2018-06-07
- Version: 0.5.0
+|
Last Published: 2018-09-12
+ Version: 0.6.0
@@ -55,20 +55,22 @@
Platform
Performance-tuning-guide
-Api
Common
Data-management
Elasticsearch
Enrichment
Indexing
+Job
Management
Parsers
3rdPartyParser
+
[5/5] metron git commit: METRON-1776: Update public web site to point at 0.6.0 new release
METRON-1776: Update public web site to point at 0.6.0 new release Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0bea5bdb Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0bea5bdb Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0bea5bdb Branch: refs/heads/asf-site Commit: 0bea5bdbefaa860216fa17afc4224a0284429733 Parents: 1a131d8 Author: justinjleet Authored: Thu Sep 13 15:48:23 2018 -0400 Committer: justinjleet Committed: Thu Sep 13 15:48:23 2018 -0400 -- current-book/CONTRIBUTING.html | 8 +- current-book/Upgrading.html | 8 +- current-book/css/print.css | 11 +- current-book/images/icon_error_sml.gif | Bin 1010 -> 633 bytes current-book/images/icon_info_sml.gif | Bin 606 -> 638 bytes current-book/images/icon_success_sml.gif| Bin 990 -> 604 bytes current-book/images/icon_warning_sml.gif| Bin 576 -> 625 bytes .../metron-job_state_statechart_diagram.svg | 14 + current-book/index.html | 8 +- current-book/metron-analytics/index.html| 8 +- .../metron-maas-service/index.html | 8 +- .../metron-profiler-client/index.html | 8 +- .../metron-analytics/metron-profiler/index.html | 53 ++- .../metron-statistics/HLLP.html | 8 +- .../metron-statistics/index.html| 8 +- .../metron-contrib/metron-docker/index.html | 8 +- .../metron-performance/index.html | 8 +- .../Kerberos-ambari-setup.html | 8 +- .../Kerberos-manual-setup.html | 97 - .../metron-deployment/amazon-ec2/index.html | 8 +- .../metron-deployment/ansible/index.html| 8 +- .../metron-deployment/ansible/roles/index.html | 8 +- .../ansible/roles/opentaxii/index.html | 8 +- .../ansible/roles/pcap_replay/index.html| 8 +- .../ansible/roles/sensor-stubs/index.html | 8 +- .../ansible/roles/sensor-test-mode/index.html | 8 +- .../development/centos6/index.html | 12 +- .../development/fastcapa/index.html | 8 +- .../metron-deployment/development/index.html| 8 +- .../development/ubuntu14/index.html | 12 +- current-book/metron-deployment/index.html | 13 +- .../metron-deployment/other-examples/index.html | 8 +- .../manual-install/Manual_Install_CentOS6.html | 8 +- .../ambari/elasticsearch-mpack/index.html | 8 +- .../packaging/ambari/index.html | 8 +- .../packaging/ambari/metron-mpack/index.html| 8 +- .../packaging/docker/ansible-docker/index.html | 8 +- .../packaging/docker/deb-docker/index.html | 8 +- .../packaging/docker/rpm-docker/index.html | 8 +- .../packaging/packer-build/index.html | 8 +- .../metron-interface/metron-alerts/index.html | 50 ++- .../metron-interface/metron-config/index.html | 8 +- .../metron-interface/metron-rest/index.html | 204 - .../Performance-tuning-guide.html | 15 +- current-book/metron-platform/index.html | 13 +- .../metron-platform/metron-common/index.html| 43 +- .../metron-data-management/index.html | 11 +- .../metron-elasticsearch/index.html | 19 +- .../metron-enrichment/Performance.html | 11 +- .../metron-enrichment/index.html| 23 +- .../metron-platform/metron-indexing/index.html | 65 ++- .../metron-platform/metron-job/index.html | 126 ++ .../metron-management/index.html| 11 +- .../metron-parsers/3rdPartyParser.html | 87 ++-- .../metron-parsers/ParserChaining.html | 259 .../metron-platform/metron-parsers/index.html | 131 -- .../metron-parsers/parser-testing.html | 12 +- .../metron-pcap-backend/index.html | 15 +- .../metron-platform/metron-solr/index.html | 300 ++ .../metron-platform/metron-writer/index.html| 33 +- current-book/metron-sensors/fastcapa/index.html | 8 +- current-book/metron-sensors/index.html | 8 +- current-book/metron-sensors/pycapa/index.html | 181 ++-- .../stellar-3rd-party-example/index.html| 8 +- .../stellar-common/3rdPartyStellar.html | 8 +- .../metron-stellar/stellar-common/index.html| 27 +- .../metron-stellar/stellar-zeppelin/index.html | 14 +- .../use-cases/forensic_clustering/index.html| 241 +++ .../geographic_login_outliers/index.html| 14 +- current-book/use-cases/index.html | 9 +- .../use-cases/parser_chaining/index.html| 412 +++ .../use-cases/typosquat_detection/index.html| 23 +- documentation/index.html| 15 +-
[1/5] metron git commit: METRON-1776 Update public web site to point at 0.6.0 new release (justinleet) closes apache/metron#1195
Repository: metron
Updated Branches:
refs/heads/master 2720e537e -> a97e575f3
http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/use-cases/parser_chaining/index.html
--
diff --git a/site/current-book/use-cases/parser_chaining/index.html
b/site/current-book/use-cases/parser_chaining/index.html
new file mode 100644
index 000..5ddcde2
--- /dev/null
+++ b/site/current-book/use-cases/parser_chaining/index.html
@@ -0,0 +1,412 @@
+
+
+http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
+
+
+
+
+
+Metron Problem Statement
+
+
+
+
+
+ $( document ).ready( function() { $( '.carousel' ).carousel( {
interval: 3500 } ) } );
+
+
+
+
+
+http://metron.apache.org/;
id="bannerLeft">
+
+
+
+
+
+
+ http://www.apache.org; class="externalLink"
title="Apache">Apache/
+ http://metron.apache.org/; class="externalLink"
title="Metron">Metron/
+ Documentation/
+Problem Statement
+|
Last Published: 2018-09-12
+ Version: 0.6.0
+
+
+
+
+
+
+ User Documentation
+Metron
+
+CONTRIBUTING
+Upgrading
+Analytics
+Docker
+Performance
+Deployment
+Alerts
+Config
+Rest
+Platform
+Sensors
+Stellar-3rd-party-example
+Stellar-common
+Stellar-zeppelin
+Use-cases
+
+Forensic_clustering
+Geographic_login_outliers
+Parser_chaining
+Typosquat_detection
+
+
+
+
+
+
+
+
+
+
+
+http://maven.apache.org/; title="Built by Maven"
class="poweredBy">
+
+
+
+
+
+Problem Statement
+
+Aggregating many different types sensors into a single data source (e.g.
syslog) and ingesting that aggregate sensor into Metron is a common pattern.
It is not obvious precisely how to manage these types of aggregate sensors as
they require two-pass parsing. This document will walk through an example of
supporting this kind of multi-pass ingest.
+Multi-pass parser involves the following requirements:
+
+
+The enveloping parser (e.g. the aggregation format such as syslog or plain
CSV) may contain metadata which should be ingested along with the data.
+The enveloping sensor contains many different sensor types
+
+
+High Level Solution
+
+At a high level, we continue to maintain the architectural invariant of a
1-1 relationship between logical sensors and storm topologies. Eventually this
relationship may become more complex, but at the moment the approach is to
construct a routing parser which will have two responsibilities:
+
+
+Parse the envelope (e.g. syslog data) and extract any metadata fields from
the envelope to pass along
+Route the unfolded data to the appropriate kafka topic associated with the
enveloped sensor data
+
+Because the data emitted from the routing parser is just like any data
emitted from any other parser, in that it is a JSON blob like any data emitted
from any parser, we will need to adjust the downstream parsers to extract the
enveloped data from the JSON blob and treat it as the data to parse.
+
+Example
+
+Preliminaries
+We assume that the following environment variables are set:
+
+
+METRON_HOME - the home directory for metron
+ZOOKEEPER - The zookeeper quorum (comma separated with port
specified: e.g. node1:2181 for full-dev)
+BROKERLIST - The Kafka broker list (comma separated with port
specified: e.g. node1:6667 for full-dev)
+ES_HOST - The elasticsearch master (and port) e.g.
node1:9200 for full-dev.
+
+Before editing configurations, be sure to pull the configs from zookeeper
locally via
+
+
+
+$METRON_HOME/bin/zk_load_configs.sh --mode PULL -z
$ZOOKEEPER -o $METRON_HOME/config/zookeeper/ -f
+
+
+
+The Scenario
+Consider the following situation, we have some logs from a Cisco PIX device
that we would like to ingest. The format is syslog, but multiple scenarios
exist in the same log file. Specificaly, lets consider the sample logs
http://www.monitorware.com/en/logsamples/cisco-pix-61(2).php">here.
+The log lines in general have the following components:
+
+
+A timestamp
+A message type tag
+The message payload that is dependent upon the tag
+
+Lets consider two types of messages that wed like to
parse:
+
+
+Tag 6-302* which are connection creation and teardown messages
e.g. Built UDP connection for faddr 198.207.223.240/53337 gaddr
10.0.0.187/53 laddr 192.168.0.2/53
+Tag 5-304* which are URL access events e.g. 192.168.0.2
Accessed URL 66.102.9.99:/
+
+A couple things are apparent from this:
+
+
+The formats we care about are easy to represent in grok, but are very
different and logically represent very different sensors.
+The syslog loglines output by this device has many types of events that I
do not care about
[2/5] metron git commit: METRON-1776 Update public web site to point at 0.6.0 new release (justinleet) closes apache/metron#1195
http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-pcap-backend/index.html
--
diff --git a/site/current-book/metron-platform/metron-pcap-backend/index.html
b/site/current-book/metron-platform/metron-pcap-backend/index.html
index 5206fd0..e1c3000 100644
--- a/site/current-book/metron-platform/metron-pcap-backend/index.html
+++ b/site/current-book/metron-platform/metron-pcap-backend/index.html
@@ -1,13 +1,13 @@
http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
-
+
Metron Metron PCAP Backend
@@ -32,8 +32,8 @@
http://metron.apache.org/; class="externalLink"
title="Metron">Metron/
Documentation/
Metron PCAP Backend
-|
Last Published: 2018-06-07
- Version: 0.5.0
+|
Last Published: 2018-09-12
+ Version: 0.6.0
@@ -55,15 +55,16 @@
Platform
Performance-tuning-guide
-Api
Common
Data-management
Elasticsearch
Enrichment
Indexing
+Job
Management
Parsers
Pcap-backend
+Solr
Writer
@@ -227,12 +228,14 @@ limitations under the License.
-nr,--num_reducers argThe number of reducers to use. Default
is 10.
-h,--help Display help
+ -ps,--print_status Print the status of the job as it runs
-ir,--include_reverse Indicates if filter should check swapped
src/dest addresses and IPs
-p,--protocol arg IP Protocol
-sa,--ip_src_addr arg Source IP address
-sp,--ip_src_port arg Source port
-st,--start_time arg (required) Packet start time range.
+ -yq,--yarn_queue arg Yarn queue this job will be submitted to
@@ -250,8 +253,10 @@ limitations under the License.
-nr,--num_reducers argThe number of reducers to use. Default
is 10.
-h,--help Display help
+ -ps,--print_status Print the status of the job as it runs
-q,--query argQuery string to use as a filter
-st,--start_time arg (required) Packet start time range.
+ -yq,--yarn_queue arg Yarn queue this job will be submitted to
The Query filters --query argument specifies the Stellar
expression to execute on each packet. To interact with the packet, a few
variables are exposed:
http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-solr/index.html
--
diff --git a/site/current-book/metron-platform/metron-solr/index.html
b/site/current-book/metron-platform/metron-solr/index.html
new file mode 100644
index 000..1afb245
--- /dev/null
+++ b/site/current-book/metron-platform/metron-solr/index.html
@@ -0,0 +1,300 @@
+
+
+http://www.w3.org/1999/xhtml; xml:lang="en" lang="en">
+
+
+
+
+
+Metron Solr in Metron
+
+
+
+
+
+ $( document ).ready( function() { $( '.carousel' ).carousel( {
interval: 3500 } ) } );
+
+
+
+
+
+http://metron.apache.org/;
id="bannerLeft">
+
+
+
+
+
+
+ http://www.apache.org; class="externalLink"
title="Apache">Apache/
+ http://metron.apache.org/; class="externalLink"
title="Metron">Metron/
+ Documentation/
+Solr in Metron
+|
Last Published: 2018-09-12
+ Version: 0.6.0
+
+
+
+
+
+
+ User Documentation
+Metron
+
+CONTRIBUTING
+Upgrading
+Analytics
+Docker
+Performance
+Deployment
+Alerts
+Config
+Rest
+Platform
+
+Performance-tuning-guide
+Common
+Data-management
+Elasticsearch
+Enrichment
+Indexing
+Job
+Management
+Parsers
+Pcap-backend
+Solr
+Writer
+
+
+Sensors
+Stellar-3rd-party-example
+Stellar-common
+Stellar-zeppelin
+Use-cases
+
+
+
+
+
+
+
+
+
+http://maven.apache.org/; title="Built by Maven"
class="poweredBy">
+
+
+
+
+
+Solr in Metron
+
+
+Table of Contents
+
+
+Introduction
+Configuration
+Installing
+Schemas
+Collections
+
+
+Introduction
+Metron ships with Solr 6.6.2 support. Solr Cloud can be used as the
real-time portion of the datastore resulting from metron-indexing.
+
+Configuration
+
+The Indexing Topology
+Solr is a viable option for the random access topology and,
similar to the Elasticsearch Writer, can be configured via the global config.
The following settings are possible as part of the global config:
+
+
+solr.zookeeper
+
+
+The zookeeper quorum associated with
[5/5] metron git commit: METRON-1776 Update public web site to point at 0.6.0 new release (justinleet) closes apache/metron#1195
METRON-1776 Update public web site to point at 0.6.0 new release (justinleet) closes apache/metron#1195 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a97e575f Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a97e575f Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a97e575f Branch: refs/heads/master Commit: a97e575f3bf6285d9893c2bc63d70a05784e22ea Parents: 2720e53 Author: justinleet Authored: Thu Sep 13 15:19:43 2018 -0400 Committer: leet Committed: Thu Sep 13 15:19:43 2018 -0400 -- site/current-book/CONTRIBUTING.html | 8 +- site/current-book/Upgrading.html| 8 +- site/current-book/css/maven-base.css| 168 site/current-book/css/maven-theme.css | 161 site/current-book/css/print.css | 11 +- site/current-book/images/collapsed.gif | Bin 53 -> 0 bytes site/current-book/images/expanded.gif | Bin 52 -> 0 bytes site/current-book/images/external.png | Bin 230 -> 0 bytes site/current-book/images/icon_error_sml.gif | Bin 1010 -> 633 bytes site/current-book/images/icon_info_sml.gif | Bin 606 -> 638 bytes site/current-book/images/icon_success_sml.gif | Bin 990 -> 604 bytes site/current-book/images/icon_warning_sml.gif | Bin 576 -> 625 bytes .../metron-job_state_statechart_diagram.svg | 14 + site/current-book/images/newwindow.png | Bin 220 -> 0 bytes site/current-book/index.html| 8 +- site/current-book/metron-analytics/index.html | 8 +- .../metron-maas-service/index.html | 8 +- .../metron-profiler-client/index.html | 8 +- .../metron-analytics/metron-profiler/index.html | 53 ++- .../metron-statistics/HLLP.html | 8 +- .../metron-statistics/index.html| 8 +- .../metron-contrib/metron-docker/index.html | 8 +- .../metron-performance/index.html | 8 +- .../Kerberos-ambari-setup.html | 8 +- .../Kerberos-manual-setup.html | 97 - .../metron-deployment/amazon-ec2/index.html | 8 +- .../metron-deployment/ansible/index.html| 8 +- .../metron-deployment/ansible/roles/index.html | 8 +- .../ansible/roles/opentaxii/index.html | 8 +- .../ansible/roles/pcap_replay/index.html| 8 +- .../ansible/roles/sensor-stubs/index.html | 8 +- .../ansible/roles/sensor-test-mode/index.html | 8 +- .../development/centos6/index.html | 12 +- .../development/fastcapa/index.html | 8 +- .../metron-deployment/development/index.html| 8 +- .../development/ubuntu14/index.html | 12 +- site/current-book/metron-deployment/index.html | 13 +- .../metron-deployment/other-examples/index.html | 8 +- .../manual-install/Manual_Install_CentOS6.html | 8 +- .../ambari/elasticsearch-mpack/index.html | 8 +- .../packaging/ambari/index.html | 8 +- .../packaging/ambari/metron-mpack/index.html| 8 +- .../packaging/docker/ansible-docker/index.html | 8 +- .../packaging/docker/deb-docker/index.html | 8 +- .../packaging/docker/rpm-docker/index.html | 8 +- .../packaging/packer-build/index.html | 8 +- .../metron-interface/metron-alerts/index.html | 50 ++- .../metron-interface/metron-config/index.html | 8 +- .../metron-interface/metron-rest/index.html | 204 - .../Performance-tuning-guide.html | 15 +- site/current-book/metron-platform/index.html| 13 +- .../metron-platform/metron-api/index.html | 161 .../metron-platform/metron-common/index.html| 43 +- .../metron-data-management/index.html | 11 +- .../metron-elasticsearch/index.html | 19 +- .../metron-enrichment/Performance.html | 11 +- .../metron-enrichment/index.html| 23 +- .../metron-platform/metron-indexing/index.html | 65 ++- .../metron-platform/metron-job/index.html | 126 ++ .../metron-management/index.html| 11 +- .../metron-parsers/3rdPartyParser.html | 87 ++-- .../metron-parsers/ParserChaining.html | 259 .../metron-platform/metron-parsers/index.html | 131 -- .../metron-parsers/parser-testing.html | 12 +- .../metron-pcap-backend/index.html | 15 +- .../metron-platform/metron-solr/index.html | 300 ++ .../metron-platform/metron-writer/index.html| 33 +- .../metron-sensors/fastcapa/index.html | 8 +- site/current-book/metron-sensors/index.html | 8 +- .../metron-sensors/pycapa/index.html| 181 ++-- .../stellar-3rd-party-example/index.html| 8 +- .../stellar-common/3rdPartySt
[4/5] metron git commit: METRON-1776 Update public web site to point at 0.6.0 new release (justinleet) closes apache/metron#1195
http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-deployment/development/index.html -- diff --git a/site/current-book/metron-deployment/development/index.html b/site/current-book/metron-deployment/development/index.html index ea99fdf..9c6f49e 100644 --- a/site/current-book/metron-deployment/development/index.html +++ b/site/current-book/metron-deployment/development/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Metron Development Environments @@ -32,8 +32,8 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Metron Development Environments -| Last Published: 2018-06-07 - Version: 0.5.0 +| Last Published: 2018-09-12 + Version: 0.6.0 http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-deployment/development/ubuntu14/index.html -- diff --git a/site/current-book/metron-deployment/development/ubuntu14/index.html b/site/current-book/metron-deployment/development/ubuntu14/index.html index 16b2d81..fb15a5e 100644 --- a/site/current-book/metron-deployment/development/ubuntu14/index.html +++ b/site/current-book/metron-deployment/development/ubuntu14/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Metron on Ubuntu 14 @@ -32,8 +32,8 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Metron on Ubuntu 14 -| Last Published: 2018-06-07 - Version: 0.5.0 +| Last Published: 2018-09-12 + Version: 0.6.0 @@ -190,8 +190,8 @@ vagrant up Navigate to the following resources to explore your newly minted Apache Metron environment. -http://node1:4201;>Metron Alerts -http://node1:8080;>Ambari +http://node1:4201;>Metron Alerts credentials: user/password +http://node1:8080;>Ambari credentials: admin/admin Connecting to the host through SSH is as simple as running the following command. http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-deployment/index.html -- diff --git a/site/current-book/metron-deployment/index.html b/site/current-book/metron-deployment/index.html index a3ffcbd..ef7ddb1 100644 --- a/site/current-book/metron-deployment/index.html +++ b/site/current-book/metron-deployment/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron @@ -32,8 +32,8 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ -| Last Published: 2018-06-07 - Version: 0.5.0 +| Last Published: 2018-09-12 + Version: 0.6.0 @@ -171,7 +171,10 @@ limitations under the License. Running Metron within the resource constraints of a single VM is incredibly challenging. Failing to respect this warning, will cause various services to fail mysteriously as the system runs into memory and processing limits. How? -To deploy Metron in a VM running on your computer, follow the instructions at development/centos6. +To deploy Metron in a VM running on your computer, follow the instructions at development/centos6. + +How do I address services crashing when running Metron on a single VM? +We recommend looking at Ambari and shutting down any services you may not be using. For example, we recommend turning off Metron Profiler, as this commonly causes REST services to crash when running on a single VM. How do I build RPM packages? This provides RPM packages that allow you to install Metron on an RPM-based operating system like CentOS. http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-deployment/other-examples/index.html -- diff --git a/site/current-book/metron-deployment/other-examples/index.html b/site/current-book/metron-deployment/other-examples/index.html index 3a89cb0..41ee390 100644 --- a/site/current-book/metron-deployment/other-examples/index.html +++ b/site/current-book/metron-deployment/other-examples/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Other Example Deployments @@ -32,8 +32,8 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Other Example Deployments -| Last Published: 2018-06-07 - Version: 0.5.0 +| Last Published:
[3/5] metron git commit: METRON-1776 Update public web site to point at 0.6.0 new release (justinleet) closes apache/metron#1195
http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-enrichment/Performance.html -- diff --git a/site/current-book/metron-platform/metron-enrichment/Performance.html b/site/current-book/metron-platform/metron-enrichment/Performance.html index 136d939..0857ee4 100644 --- a/site/current-book/metron-platform/metron-enrichment/Performance.html +++ b/site/current-book/metron-platform/metron-enrichment/Performance.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Enrichment Performance @@ -32,8 +32,8 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Enrichment Performance -| Last Published: 2018-06-07 - Version: 0.5.0 +| Last Published: 2018-09-12 + Version: 0.6.0 @@ -55,7 +55,6 @@ Platform Performance-tuning-guide -Api Common Data-management Elasticsearch @@ -65,9 +64,11 @@ Indexing +Job Management Parsers Pcap-backend +Solr Writer http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-enrichment/index.html -- diff --git a/site/current-book/metron-platform/metron-enrichment/index.html b/site/current-book/metron-platform/metron-enrichment/index.html index de1e4fa..e750946 100644 --- a/site/current-book/metron-platform/metron-enrichment/index.html +++ b/site/current-book/metron-platform/metron-enrichment/index.html @@ -1,13 +1,13 @@ http://www.w3.org/1999/xhtml; xml:lang="en" lang="en"> - + Metron Enrichment @@ -32,8 +32,8 @@ http://metron.apache.org/; class="externalLink" title="Metron">Metron/ Documentation/ Enrichment -| Last Published: 2018-06-07 - Version: 0.5.0 +| Last Published: 2018-09-12 + Version: 0.6.0 @@ -55,7 +55,6 @@ Platform Performance-tuning-guide -Api Common Data-management Elasticsearch @@ -65,9 +64,11 @@ Indexing +Job Management Parsers Pcap-backend +Solr Writer @@ -162,14 +163,22 @@ limitations under the License. There are two types of configurations at the moment, global and sensor specific. Global Configuration -There are a few enrichments which have independent configurations, such as from the global config. +There are a few enrichments which have independent configurations, such as from the global config. You can also configure the enrichment topologys writer batching settings. Also, see the Global Configuration section for more discussion of the global config. GeoIP Metron supports enrichment of IP information using https://dev.maxmind.com/geoip/geoip2/geolite2/;>GeoLite2. The location of the file is managed in the global config. geo.hdfs.file -The location on HDFS of the GeoLite2 database file to use for GeoIP lookups. This file will be localized on the storm supervisors running the topology and used from there. This is lazy, so if this property changes in a running topology, the file will be localized from HDFS upon first time the file is used via the geo enrichment. +The location on HDFS of the GeoLite2 database file to use for GeoIP lookups. This file will be localized on the storm supervisors running the topology and used from there. This is lazy, so if this property changes in a running topology, the file will be localized from HDFS upon first time the file is used via the geo enrichment. + +Writer Batching + +enrichment.writer.batchSize +The size of the batch that is written to Kafka at once. Defaults to 15 (size of 1 disables batching). + +enrichment.writer.batchTimeout +The timeout after which a batch will be flushed even if batchSize has not been met. Optional. If unspecified, or set to 0, it defaults to a system-determined duration which is a fraction of the Storm parameter topology.message.timeout.secs. Ignored if batchSize is 1, since this disables batching. Sensor Enrichment Configuration The sensor specific configuration is intended to configure the individual enrichments and threat intelligence enrichments for a given sensor type (e.g. snort). http://git-wip-us.apache.org/repos/asf/metron/blob/a97e575f/site/current-book/metron-platform/metron-indexing/index.html -- diff --git a/site/current-book/metron-platform/metron-indexing/index.html b/site/current-book/metron-platform/metron-indexing/index.html index 6d4b8d6..65c0bad 100644 --- a/site/current-book/metron-platform/metron-indexing/index.html +++
svn commit: r29345 [2/2] - /release/metron/0.6.0/
Added: release/metron/0.6.0/LICENSE == --- release/metron/0.6.0/LICENSE (added) +++ release/metron/0.6.0/LICENSE Wed Sep 12 19:09:34 2018 @@ -0,0 +1,953 @@ +Apache License + Version 2.0, January 2004 +http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a
[metron-bro-plugin-kafka] Git Push Summary
Repository: metron-bro-plugin-kafka Updated Tags: refs/tags/apache-metron-bro-plugin-kafka_0.2.0-release [created] 1dfc5239f
[metron] Git Push Summary
Repository: metron Updated Tags: refs/tags/apache-metron-0.6.0-release [created] 62310743a
[metron] Git Push Summary
Repository: metron Updated Tags: refs/tags/apache-metron-0.6.0-rc1 [created] 62310743a
[metron-bro-plugin-kafka] Git Push Summary
Repository: metron-bro-plugin-kafka Updated Tags: refs/tags/apache-metron-bro-plugin-kafka_0.2.0-rc1 [created] 1dfc5239f
svn commit: r29176 [1/2] - /dev/metron/0.6.0-RC1/
Author: leet Date: Thu Sep 6 22:35:29 2018 New Revision: 29176 Log: Adding artifacts for Metron 0.6.0-RC1 Added: dev/metron/0.6.0-RC1/ dev/metron/0.6.0-RC1/CHANGES dev/metron/0.6.0-RC1/CHANGES.bro-plugin dev/metron/0.6.0-RC1/KEYS dev/metron/0.6.0-RC1/LICENSE dev/metron/0.6.0-RC1/NOTICE dev/metron/0.6.0-RC1/apache-metron-0.6.0-rc1.tar.gz (with props) dev/metron/0.6.0-RC1/apache-metron-0.6.0-rc1.tar.gz.asc dev/metron/0.6.0-RC1/apache-metron-0.6.0-rc1.tar.gz.sha256 dev/metron/0.6.0-RC1/apache-metron-0.6.0-rc1.tar.gz.sha512 dev/metron/0.6.0-RC1/apache-metron-bro-plugin-kafka_0.2.0-rc1.tar.gz (with props) dev/metron/0.6.0-RC1/apache-metron-bro-plugin-kafka_0.2.0-rc1.tar.gz.asc dev/metron/0.6.0-RC1/apache-metron-bro-plugin-kafka_0.2.0-rc1.tar.gz.sha256 dev/metron/0.6.0-RC1/apache-metron-bro-plugin-kafka_0.2.0-rc1.tar.gz.sha512 Added: dev/metron/0.6.0-RC1/CHANGES == --- dev/metron/0.6.0-RC1/CHANGES (added) +++ dev/metron/0.6.0-RC1/CHANGES Thu Sep 6 22:35:29 2018 @@ -0,0 +1,135 @@ +METRON-1764 Update version to 0.6.0 (justinleet) closes apache/metron#1183 +METRON-1751 Storm Profiler dies when consuming null message (nickwallen) closes apache/metron#1176 +METRON-1757 Storm Profiler Serialization Exception (nickwallen) closes apache/metron#1178 +METRON-1743 CEF testPaloAltoCEF test using a confusing variable name (JonZeolla via justinleet) closes apache/metron#1173 +METRON-1752 Prevent package.lock from changing during build (sardell via merrimanr) closes apache/metron#1177 +METRON-1724 Date/time validation missing in PCAP query (tiborm via nickwallen) closes apache/metron#1172 +METRON-1739 UDP packets are not handled (merrimanr) closes apache/metron#1168 +METRON-1727: Alerts are not populated on the alerts UI after enabling X-pack for Elastic search (MohanDV via mmiklavc) closes apache/metron#1141 +METRON-1738: Pcap directories should have correct permissions (merrimanr via mmiklavc) closes apache/metron#1166 +METRON-1737: Document Job cleanup (merrimanr via mmiklavc) closes apache/metron#1164 +METRON-1732: Fix job status liveness bug and parallelize finalizer file writing (mmiklavc via mmiklavc) closes apache/metron#1157 +METRON-1735 Empty print status option causes NPE (merrimanr) closes apache/metron#1160 +METRON-1733 PCAP UI - PCAP queries dont work on Safari (sardell via merrimanr) closes apache/metron#1158 +METRON-1734 Src and Dst port filters are incorrect after changing to empty (merrimanr) closes apache/metron#1159 +METRON-1725 Add ability to specify YARN queue for pcap jobs (merrimanr) closes apache/metron#1153 +METRON-1731: PCAP - Escape colons in output dir names (mmiklavc via mmiklavc) closes apache/metron#1155 +METRON-1702 Reload a running job in the UI (merrimanr) closes apache/metron#1156 +METRON-1722 PcapCLI should print progress to stdout (merrimanr) closes apache/metron#1138 +METRON-1728: Handle null values in config in Pcap backend more gracefully (mmiklavc via mmiklavc) closes apache/metron#1151 +METRON-1730: Update steps to run pycapa on Centos 6 (mmiklavc via mmiklavc) closes apache/metron#1152 +METRON-1713 PCAP UI - Add a way to kill a pcap job (tiborm via merrimanr) closes apache/metron#1143 +METRON-1723 PCAP UI - Unable to select/copy from packets details in PCAP query panel (sardell via merrimanr) closes apache/metron#1139 +METRON-1712 PCAP UI - Input validation (tiborm via merrimanr) closes apache/metron#1142 +METRON-1720 Better error messages when there are no results or wireshark is not installed (merrimanr) closes apache/metron#1154 +METRON-1726: Refactor PcapTopologyIntegrationTest (mmiklavc via mmiklavc) closes apache/metron#1140 +METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr) closes apache/metron#1122 +METRON-1675 PCAP UI - Introduce the paging capability (sardell via merrimanr) closes apache/metron#1121 +METRON-1721 New default input path is wrong in pcap CLI (merrimanr) closes apache/metron#1137 +METRON-1676 PCAP UI - Add data range selector to the filter bar (tiborm via merrimanr) closes apache/metron#1119 +METRON-1662 PCAP UI - Downloading PCAP page files (tiborm via merrimanr) closes apache/metron#1118 +METRON-1700 Create REST endpoint to get job configuration (merrimanr) closes apache/metron#1135 +METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103 +METRON-1701 Update General notes on the installation of Pycapa on Kerberized cluster (MohanDV via nickwallen) closes apache/metron#1136 +METRON-1650 Packaging docker containers are too large (jameslamb via merrimanr) closes apache/metron#1091 +METRON-1604 : Add RHEL 7 power pc to OS family for the HCP management pack repo info closes apache/incubator-metron#1052
svn commit: r29176 [2/2] - /dev/metron/0.6.0-RC1/
Added: dev/metron/0.6.0-RC1/LICENSE == --- dev/metron/0.6.0-RC1/LICENSE (added) +++ dev/metron/0.6.0-RC1/LICENSE Thu Sep 6 22:35:29 2018 @@ -0,0 +1,953 @@ +Apache License + Version 2.0, January 2004 +http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a
[metron-bro-plugin-kafka] Git Push Summary
Repository: metron-bro-plugin-kafka Updated Branches: refs/heads/Metron_bro-plugin-kafka_0.2.0 [created] 1dfc5239f
[metron] Git Push Summary
Repository: metron Updated Branches: refs/heads/Metron_0.6.0 [created] 62310743a
metron git commit: METRON-1764 Update version to 0.6.0 (justinleet) closes apache/metron#1183
Repository: metron Updated Branches: refs/heads/master d32bd50d4 -> 62310743a METRON-1764 Update version to 0.6.0 (justinleet) closes apache/metron#1183 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/62310743 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/62310743 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/62310743 Branch: refs/heads/master Commit: 62310743a0c1d4a27db47a89cf79496d594c8bcf Parents: d32bd50 Author: justinleet Authored: Tue Sep 4 19:53:37 2018 -0400 Committer: leet Committed: Tue Sep 4 19:53:37 2018 -0400 -- metron-analytics/metron-maas-common/pom.xml| 2 +- metron-analytics/metron-maas-service/pom.xml | 2 +- metron-analytics/metron-profiler-client/pom.xml| 2 +- metron-analytics/metron-profiler-common/pom.xml| 2 +- metron-analytics/metron-profiler/pom.xml | 2 +- metron-analytics/metron-statistics/pom.xml | 2 +- metron-analytics/pom.xml | 2 +- metron-contrib/metron-docker/pom.xml | 2 +- metron-contrib/metron-performance/pom.xml | 2 +- metron-contrib/pom.xml | 2 +- metron-deployment/Kerberos-manual-setup.md | 4 ++-- metron-deployment/amazon-ec2/conf/defaults.yml | 2 +- metron-deployment/ansible/playbooks/docker_probe_install.yml | 2 +- .../ansible/roles/ambari_master/defaults/main.yml | 2 +- .../development/centos6/ansible/inventory/group_vars/all | 2 +- .../development/ubuntu14/ansible/inventory/group_vars/all | 2 +- metron-deployment/packaging/ambari/elasticsearch-mpack/pom.xml | 4 ++-- metron-deployment/packaging/ambari/metron-mpack/pom.xml| 4 ++-- .../ambari/metron-mpack/src/main/resources/mpack.json | 2 +- metron-deployment/packaging/docker/deb-docker/pom.xml | 4 ++-- metron-deployment/packaging/docker/rpm-docker/pom.xml | 4 ++-- metron-deployment/pom.xml | 2 +- metron-interface/metron-alerts/package-lock.json | 2 +- metron-interface/metron-alerts/package.json| 2 +- metron-interface/metron-alerts/pom.xml | 2 +- metron-interface/metron-config/package-lock.json | 2 +- metron-interface/metron-config/package.json| 2 +- metron-interface/metron-config/pom.xml | 2 +- metron-interface/metron-config/scripts/package.json| 2 +- metron-interface/metron-rest-client/pom.xml| 2 +- metron-interface/metron-rest/pom.xml | 2 +- .../metron-rest/src/main/resources/application.yml | 2 +- metron-interface/pom.xml | 2 +- metron-platform/Performance-tuning-guide.md| 4 ++-- metron-platform/README.md | 2 +- metron-platform/elasticsearch-shaded/pom.xml | 2 +- metron-platform/metron-common/pom.xml | 2 +- metron-platform/metron-common/src/main/scripts/cluster_info.py | 2 +- metron-platform/metron-data-management/pom.xml | 2 +- metron-platform/metron-elasticsearch/pom.xml | 2 +- metron-platform/metron-enrichment/pom.xml | 2 +- metron-platform/metron-hbase-client/pom.xml| 2 +- metron-platform/metron-hbase/pom.xml | 2 +- metron-platform/metron-indexing/pom.xml| 2 +- metron-platform/metron-integration-test/pom.xml| 2 +- metron-platform/metron-job/pom.xml | 2 +- metron-platform/metron-management/pom.xml | 2 +- metron-platform/metron-parsers/pom.xml | 2 +- metron-platform/metron-pcap-backend/pom.xml| 2 +- .../metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh | 2 +- metron-platform/metron-pcap/pom.xml| 2 +- metron-platform/metron-solr/pom.xml| 2 +- metron-platform/metron-storm-kafka-override/pom.xml| 2 +- metron-platform/metron-storm-kafka/pom.xml | 2 +- metron-platform/metron-test-utilities/pom.xml | 2 +- metron-platform/metron-writer/pom.xml | 2 +- metron-platform/metron-zookeeper/pom.xml | 2 +- metron-platform/pom.xml| 2 +- metron-stellar/pom.xml | 2 +- metron-stellar/stellar-3rd-party-example/pom.xml | 2 +- metron-stel
metron git commit: METRON-1672 Add metron-alerts s UI unit tests to travis build process (justinleet) closes apache/metron#1106
Repository: metron
Updated Branches:
refs/heads/master 52bd310fc -> aceca469a
METRON-1672 Add metron-alertss UI unit tests to travis build process
(justinleet) closes apache/metron#1106
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/aceca469
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/aceca469
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/aceca469
Branch: refs/heads/master
Commit: aceca469a83d17c033ea7a47ff82c61962319b3a
Parents: 52bd310
Author: justinleet
Authored: Thu Jul 19 09:30:59 2018 -0400
Committer: leet
Committed: Thu Jul 19 09:30:59 2018 -0400
--
.travis.yml | 14 +++---
metron-interface/metron-alerts/karma.conf.js | 15 ++-
metron-interface/metron-alerts/package.json | 3 ++-
metron-interface/metron-alerts/pom.xml | 14 --
4 files changed, 39 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/aceca469/.travis.yml
--
diff --git a/.travis.yml b/.travis.yml
index 71d7165..3a73d7e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,9 +16,14 @@
dist: trusty
sudo: required
+addons:
+ chrome: stable
env:
- - TEST_TYPE=default
+ - SCRIPT="mvn surefire:test@unit-tests -T 2C"
+ - SCRIPT="mvn surefire:test@integration-tests"
+ - SCRIPT="mvn test --projects
metron-interface/metron-config,metron-interface/metron-alerts"
+ - SCRIPT="./dev-utilities/build-utils/verify_licenses.sh"
install: true
language: java
@@ -33,10 +38,13 @@ before_install:
- npm config set prefix $HOME/.npm-prefix --global
install:
- - time mvn -q -T 2C -DskipTests clean install
+ - time mvn install -T 2C -q -DskipTests=true -Dmaven.javadoc.skip=true -B -V
script:
- - time mvn -q -T 2C surefire:test@unit-tests && time mvn -q
surefire:test@integration-tests && time mvn -q test --projects
metron-interface/metron-config && time
dev-utilities/build-utils/verify_licenses.sh
+ - time $SCRIPT
+
+matrix:
+ fast_finish: true
before_cache:
- rm -rf $HOME/.m2/repository/org/apache/metron
http://git-wip-us.apache.org/repos/asf/metron/blob/aceca469/metron-interface/metron-alerts/karma.conf.js
--
diff --git a/metron-interface/metron-alerts/karma.conf.js
b/metron-interface/metron-alerts/karma.conf.js
index f245e1d..7f29c92 100644
--- a/metron-interface/metron-alerts/karma.conf.js
+++ b/metron-interface/metron-alerts/karma.conf.js
@@ -18,6 +18,8 @@
// Karma configuration file, see link for more information
// https://karma-runner.github.io/0.13/config/configuration-file.html
+process.env.CHROME_BIN = require('puppeteer').executablePath()
+
module.exports = function (config) {
config.set({
basePath: '',
@@ -55,7 +57,18 @@ module.exports = function (config) {
colors: true,
logLevel: config.LOG_INFO,
autoWatch: true,
-browsers: ['Chrome'],
+browsers: ['Chrome','ChromeHeadless'],
+customLaunchers: {
+ ChromeHeadless: {
+base: 'Chrome',
+flags: [
+ '--no-sandbox',
+ '--headless',
+ '--disable-gpu',
+ '--remote-debugging-port=9222'
+]
+ }
+},
singleRun: false
});
};
http://git-wip-us.apache.org/repos/asf/metron/blob/aceca469/metron-interface/metron-alerts/package.json
--
diff --git a/metron-interface/metron-alerts/package.json
b/metron-interface/metron-alerts/package.json
index 0a40f0d..9cfc09f 100644
--- a/metron-interface/metron-alerts/package.json
+++ b/metron-interface/metron-alerts/package.json
@@ -7,7 +7,7 @@
"build": "./node_modules/@angular/cli/bin/ng build -prod",
"start": "ng serve",
"lint": "tslint \"src/**/*.ts\"",
-"test": "./node_modules/@angular/cli/bin/ng test --watch=false",
+"test": "karma start --single-run --browsers ChromeHeadless karma.conf.js",
"pree2e": "webdriver-manager update",
"e2e": "protractor-flake --protractor-path=./node_modules/.bin/protractor
--max-attempts=3 -- ./protractor.conf.js"
},
@@ -58,6 +58,7 @@
"optimist": "0.6.1",
"protractor": "5.3.0",
"protractor-flake": "^3.3.0",
+"puppeteer": "^1.6.0",
"serve-favicon": "2.4.2",
"serve-static": "1.12.1",
"ssh2": "^0.5.5",
http://git-wip-us.
metron git commit: METRON-1684 Fix Markdown problems in 3rdPartyParser.md (justinleet) closes apache/metron#1110
Repository: metron
Updated Branches:
refs/heads/master f43453830 -> 52bd310fc
METRON-1684 Fix Markdown problems in 3rdPartyParser.md (justinleet) closes
apache/metron#1110
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/52bd310f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/52bd310f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/52bd310f
Branch: refs/heads/master
Commit: 52bd310fcce68dad15eead57f1113092a30d9791
Parents: f434538
Author: justinleet
Authored: Wed Jul 18 13:18:49 2018 -0400
Committer: leet
Committed: Wed Jul 18 13:18:49 2018 -0400
--
.../metron-parsers/3rdPartyParser.md| 408 +--
1 file changed, 204 insertions(+), 204 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/52bd310f/metron-platform/metron-parsers/3rdPartyParser.md
--
diff --git a/metron-platform/metron-parsers/3rdPartyParser.md
b/metron-platform/metron-parsers/3rdPartyParser.md
index 739328f..0c2409e 100644
--- a/metron-platform/metron-parsers/3rdPartyParser.md
+++ b/metron-platform/metron-parsers/3rdPartyParser.md
@@ -54,180 +54,180 @@ project. We'll call it `extra_parsers`, so in your
workspace, let's set
up the maven project:
* Create the maven infrastructure for `extra_parsers` via
- ```
- mkdir -p extra_parsers/src/{main,test}/java
- ```
+```
+mkdir -p extra_parsers/src/{main,test}/java
+```
* Create a pom file indicating how we should build our parsers by
editing `extra_parsers/pom.xml` with the following content:
- ```
- http://maven.apache.org/POM/4.0.0;
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
-xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd;>
-4.0.0
-com.3rdparty
-extra-parsers
-jar
-1.0-SNAPSHOT
-extra-parsers
-http://thirdpartysoftware.org
-
-
- 1.8
-
- 0.4.1
-
- 19.0
-
- 2.4.3
-
-
-
-
-org.apache.metron
-metron-parsers
-${metron_version}
-provided
-
-
-com.google.guava
-guava
-${guava_version}
-
-
-junit
-junit
-3.8.1
-test
-
-
-
-
-
+1.8
+
+0.4.1
+
+19.0
+
+2.4.3
+
+
+
-
-
- org.apache.maven.plugins
- maven-shade-plugin
- ${shade_version}
-
-true
-
-
-
-*slf4j*
-
-
-
-
-
- package
-
-shade
-
-
-true
-uber
-
-
-
-*:*
+
+ org.apache.metron
+ metron-parsers
+ ${metron_version}
+ provided
+
+
+ com.google.guava
+ guava
+ ${guava_version}
+
+
+ junit
+ junit
+ 3.8.1
+ test
+
+
+
+
+
+
+
+org.apache.maven.plugins
+maven-shade-plugin
+${shade_version}
+
+ true
+
+
+
+ *slf4j*
+
+
+
+
+
+package
+
+ shade
+
+
+ true
+ uber
+
+
+
+ *:*
+
+META-INF/*.SF
+META-INF/*.DSA
+META-INF/*.RSA
+
+
+
+
+
+
+ com.google
+ com.thirdparty.guava
+
+
+
- META-INF/*.SF
- META-INF/*.DSA
- META-INF/*.RSA
+
+ storm:storm-core:*
+ storm:storm-lib:*
+ org.slf4j.impl*
+ org.slf4j:slf4j-log4j*
-
-
-
-
-
-com.google
-com.
[2/3] metron git commit: METRON-1657 Parser aggregation in storm (justinleet) closes apache/metron#1099
http://git-wip-us.apache.org/repos/asf/metron/blob/f4345383/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
--
diff --git
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
index 15ce735..06f4cec 100644
---
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
+++
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bolt/ParserBoltTest.java
@@ -58,6 +58,7 @@ import
org.apache.metron.common.zookeeper.configurations.ConfigurationsUpdater;
import org.apache.metron.parsers.BasicParser;
import org.apache.metron.parsers.interfaces.MessageFilter;
import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.topology.ParserComponents;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.test.bolt.BaseBoltTest;
import org.apache.metron.test.error.MetronErrorJSONMatcher;
@@ -185,7 +186,15 @@ public class ParserBoltTest extends BaseBoltTest {
@Test
public void testEmpty() throws Exception {
String sensorType = "yaf";
-ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser,
new WriterHandler(writer)) {
+Map parserMap = Collections.singletonMap(
+sensorType,
+new ParserComponents(
+parser,
+null,
+new WriterHandler(writer)
+)
+);
+ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserMap) {
@Override
protected ConfigurationsUpdater createUpdater() {
return ParserBoltTest.createUpdater();
@@ -209,7 +218,7 @@ public class ParserBoltTest extends BaseBoltTest {
MetronError error = new MetronError()
.withErrorType(Constants.ErrorType.PARSER_ERROR)
.withThrowable(new NullPointerException())
-.withSensorType(sensorType)
+.withSensorType(Collections.singleton(sensorType))
.addRawMessage(sampleBinary);
verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM),
argThat(new MetronErrorJSONMatcher(error.getJSONObject(;
}
@@ -217,7 +226,15 @@ public class ParserBoltTest extends BaseBoltTest {
@Test
public void testInvalid() throws Exception {
String sensorType = "yaf";
-ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser,
new WriterHandler(writer)) {
+Map parserMap = Collections.singletonMap(
+sensorType,
+new ParserComponents(
+parser,
+null,
+new WriterHandler(writer)
+)
+);
+ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserMap) {
@Override
protected ConfigurationsUpdater createUpdater() {
return ParserBoltTest.createUpdater();
@@ -243,7 +260,7 @@ public class ParserBoltTest extends BaseBoltTest {
MetronError error = new MetronError()
.withErrorType(Constants.ErrorType.PARSER_INVALID)
-.withSensorType(sensorType)
+.withSensorType(Collections.singleton(sensorType))
.withErrorFields(new HashSet() {{ add("field"); }})
.addRawMessage(new JSONObject(){{
put("field", "invalidValue");
@@ -255,14 +272,20 @@ public class ParserBoltTest extends BaseBoltTest {
@Test
public void test() throws Exception {
-
String sensorType = "yaf";
-ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser,
new WriterHandler(writer)) {
+Map parserMap = Collections.singletonMap(
+sensorType,
+new ParserComponents(
+parser,
+null,
+new WriterHandler(writer)
+)
+);
+ParserBolt parserBolt = new ParserBolt("zookeeperUrl", parserMap) {
@Override
protected ConfigurationsUpdater createUpdater() {
return ParserBoltTest.createUpdater();
}
-
};
parserBolt.setCuratorFramework(client);
parserBolt.setZKCache(cache);
@@ -290,7 +313,6 @@ public class ParserBoltTest extends BaseBoltTest {
when(parser.validate(eq(messages.get(1.thenReturn(true);
when(filter.emitTuple(eq(messages.get(0)), any())).thenReturn(false);
when(filter.emitTuple(eq(messages.get(1)), any())).thenReturn(true);
-parserBolt.withMessageFilter(filter);
parserBolt.execute(tuple);
verify(writer, times(1)).write(eq(sensorType),
any(ParserWriterConfiguration.class), eq(tuple), eq(finalMessage2));
verify(outputCollector, times(2)).ack(tuple);
@@ -317,21 +339,15 @@ public class ParserBoltTest extends BaseBoltTest {
@Test
public void testFilterSuccess() throws Exception {
String sensorType = "yaf";
-
-ParserBolt parserBolt = new ParserBolt("zookeeperUrl", sensorType, parser,
new
[3/3] metron git commit: METRON-1657 Parser aggregation in storm (justinleet) closes apache/metron#1099
METRON-1657 Parser aggregation in storm (justinleet) closes apache/metron#1099
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f4345383
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f4345383
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f4345383
Branch: refs/heads/master
Commit: f43453830469b7fad14b59c95ad8122f320a4b05
Parents: 3a2675e
Author: justinleet
Authored: Wed Jul 18 08:54:43 2018 -0400
Committer: leet
Committed: Wed Jul 18 08:54:43 2018 -0400
--
.../service/impl/StormAdminServiceImpl.java | 23 +-
.../rest/service/impl/StormCLIWrapper.java | 25 +-
.../common/bolt/ConfiguredParserBolt.java | 10 +-
.../configuration/SensorParserConfig.java | 6 +
.../apache/metron/common/error/MetronError.java | 64 ++--
.../common/bolt/ConfiguredParserBoltTest.java | 2 +-
.../metron/common/error/MetronErrorTest.java| 16 +-
metron-platform/metron-parsers/README.md| 25 +-
.../apache/metron/parsers/bolt/ParserBolt.java | 241 --
.../apache/metron/parsers/bolt/WriterBolt.java | 3 +-
.../parsers/topology/ParserComponents.java | 67
.../parsers/topology/ParserTopologyBuilder.java | 200
.../parsers/topology/ParserTopologyCLI.java | 264 ++-
.../parsers/topology/config/ValueSupplier.java | 3 +-
.../metron/parsers/bolt/ParserBoltTest.java | 182 +++
.../metron/parsers/bolt/WriterBoltTest.java | 30 +-
.../parsers/integration/ParserDriver.java | 50 ++-
.../components/ParserTopologyComponent.java | 58 ++--
.../parsers/topology/ParserTopologyCLITest.java | 318 +--
...pleHbaseEnrichmentWriterIntegrationTest.java | 3 +-
.../integration/WriterBoltIntegrationTest.java | 150 -
.../metron/writer/BulkWriterComponent.java | 3 +-
.../metron/writer/BulkWriterComponentTest.java | 34 +-
use-cases/parser_chaining/README.md | 14 +
.../aggregated_parser_chaining_flow.svg | 14 +
.../aggregated_parser_chaining_flow.xml | 14 +
26 files changed, 1259 insertions(+), 560 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/f4345383/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
index 40b01f1..4569a23 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
@@ -50,17 +50,28 @@ public class StormAdminServiceImpl implements
StormAdminService {
TopologyResponse topologyResponse = new TopologyResponse();
if (globalConfigService.get() == null) {
topologyResponse.setErrorMessage(TopologyStatusCode.GLOBAL_CONFIG_MISSING.toString());
-} else if (sensorParserConfigService.findOne(name) == null) {
-
topologyResponse.setErrorMessage(TopologyStatusCode.SENSOR_PARSER_CONFIG_MISSING.toString());
-} else {
-topologyResponse =
createResponse(stormCLIClientWrapper.startParserTopology(name),
TopologyStatusCode.STARTED, TopologyStatusCode.START_ERROR);
+return topologyResponse;
}
-return topologyResponse;
+
+String[] sensorTypes = name.split(",");
+for (String sensorType : sensorTypes) {
+if (sensorParserConfigService.findOne(sensorType.trim()) == null) {
+topologyResponse
+
.setErrorMessage(TopologyStatusCode.SENSOR_PARSER_CONFIG_MISSING.toString());
+return topologyResponse;
+}
+}
+
+return createResponse(
+stormCLIClientWrapper.startParserTopology(name),
+TopologyStatusCode.STARTED,
+TopologyStatusCode.START_ERROR
+);
}
@Override
public TopologyResponse stopParserTopology(String name, boolean stopNow)
throws RestException {
-return createResponse(stormCLIClientWrapper.stopParserTopology(name,
stopNow), TopologyStatusCode.STOPPED, TopologyStatusCode.STOP_ERROR);
+return
createResponse(stormCLIClientWrapper.stopParserTopology(name.replaceAll(",",
"__"), stopNow), TopologyStatusCode.STOPPED, TopologyStatusCode.STOP_ERROR);
}
@Override
http://git-wip-us.apache.org/repos/asf/metron/blob/f4345383/metron-interface/metron-rest/src/main/java/
[1/3] metron git commit: METRON-1657 Parser aggregation in storm (justinleet) closes apache/metron#1099
Repository: metron Updated Branches: refs/heads/master 3a2675e5f -> f43453830 http://git-wip-us.apache.org/repos/asf/metron/blob/f4345383/use-cases/parser_chaining/aggregated_parser_chaining_flow.svg -- diff --git a/use-cases/parser_chaining/aggregated_parser_chaining_flow.svg b/use-cases/parser_chaining/aggregated_parser_chaining_flow.svg new file mode 100644 index 000..28529c8 --- /dev/null +++ b/use-cases/parser_chaining/aggregated_parser_chaining_flow.svg @@ -0,0 +1,14 @@ + +http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd;> +http://www.w3.org/2000/svg; xmlns:xlink="http://www.w3.org/1999/xlink; width="787px" height="967px" version="1.1" content="mxfile userAgent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36 version=8.9.5 editor=www.draw.io type=devicediagram name=Page-1 id=c7558073-3199-34d8-9f00-42111426c3f37Vtbb6M6EP41eewKMBDy2Nt2pXOr1COd3acVAYdYJRiB06b7648dbMAXEppCk27al+KxMXi+b8YzYzIB16vNXRHmy79wDNOJY8WbCbiZOI5tzyz6j0leKokf2JUgKVDMBzWCB/QLciG/L1mjGJbSQIJxSlAuCyOcZTAikiwsCvwsD1vgVH5qHiZQEzxEYapL/0MxWVbSwJk28m8QJUvxZMviLz4Po8ekwOuMP2/igMX2r+pehWIuPr5chjF+bonA7QRcFxiT6mq1uYYp061QW3Xf147e+r0LmJE+N4DqhqcwXfOl/42+Iv5u5EXo43mJCHzIw4i1nynmE3AVpijJaDOij4IFFTzBgiCqw0vesUJxzCa4WuCMPPDJLNouSYEfa8UCNgKl6TVOMZ3mJsMZu0lfCV8cewzctER8ZXcQryApXugQ3gu86g5OQsB1/twgKni6bIHpc1nIOZTU8zZ6pBdclR04uJpec7T5Wb6UKU5+Un4wjTk+ZYfNFPJAcLFiGs9pT0t+CjiU9GkoS6jAH ggV1/2yFxgbTEdCRliqGZoSZiVm0KT0mVdzdpWwqz/CxWO49UI5ijRYqBXn7DJ6SVEWb0EwITVnfgHGf85Trs3GW/yzJvRWyOVpOIfpVd2pIGIAaUlWYs4KVNFPmWRt/4xwd0BnALgTTdv3JSxtR8fSNljZzB8CS93MNGxgTD07b3JtZfi2JXyt0uAGke/8Pnb9g1kTZTRtZXQB37lxbRtNn6DIIoWbS7ZD1epnL7hb+XQ9eF1E9R7J1UnCIoFinGtGqQ2DZYChFhYwDQl6kt/EBA5/xj1G9B0boxaelNNgqphqtQB+U3s/Uudx5Xl8IM9TrVmbZ8uUetk9yeMfgTzMBRtHDkYPYCCHZU2Oyg7bkR2+6sj7sqMOm5SNYwR2CD+2ix0mH9/yxGGZV7HpAm1gvNet99qNu/gzgCevbU14cuHZWywBJpbUwjf58jqQ2KFwk5XALOaXN1EaliXdniUU2kpsovFdVtul48a9N97+h/D2fFPgXXxbsI2wHuj2XYNhVyzdadieATFvIKevWKNqjn3NOpDnqdsjmLUh4RHBHg10MxHuRaiM8IV3AViEUXXTqdsjPkPE/iGiZ0v4eo4h3h8tRnRMMWIX4j5F3PlE/K2IewFQUrx3xtzTMJct+qMm3m+zQzCTo7DgPfNuRw+3ZZs7S0y8qXVMTKa/Q8ylJ+KGpL2Vs+GCLHGCszBtZXKHRmVeYAjLnP3Z+Ihhmafk0K4an/eNy7SJxgzMgpOnogjxB6HbLmIPSkVwTCrW84gUYabUc3pn/kqOUMcTY1BxdipU7F8/Oiw3PZBoxhpTFXOdSCoa2MOkolN1mx2QZqBHhem8aWaqeFRh5NH82UwpZDpKutLbn6kTzUYkmh7lwaxA0XJF1V9+prW9g3UgdljhZUzB+mhJLdC3JUPRooHxjmqgWl7JDpstmk3l24RKGachTBVEZD0XsES/wnmdK+WMltu1eFcT74blXWuCy+ozDruVhqVwQQxJGG VWk0z9yxo3F+4OtzMAdq6SaE0DT8fOMWCnFjUP+xxAP3SWChLWCpZlmECDNZ4HPJ7yRYBnsCzXgI4aBh6Gjn70IFnW2aPjK2HRO8OjV/Muk6SASUhgTOWiYHQvPN0cp6QHMPLu0b3h9K8pmTbCotoIOfh7drvxIFT3LjvQT/eMe9cwEHYV/6Qjls69a6/V/WYnsVMZK+AGGlaBASq1PHMYVHq0+Jow48ygcv1jQnUyNbNXZJA9vp6Sc0t7zPKtOCmSv6U6ao4J7JlyeOb7B2aZwNemspWpBswz3Y9YN+ss6h6Hjr6JjtNj0tENbJVDakjQ+zwBqFN56neCA9LR6/Fx6MnRsSGdLRfYvlh1+x4WiCqChaJ7zsFG46lr4ik4Kk+VCrBteQeyNFAnUo/PhuSontp8JI4Gr+aowc+OxtHpJ0cH4ejbcrczrzuqJ6DvW3f09AzB9Dukz5SuXSnxHQkxxzFUSixHR6wWvgIy2mx+7FdZZ/OLSnD7Pw==/diagram/mxfile" style="background-color: rgb(255, 255, 255);">http://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inlin e-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 22px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">NiFiNiFihttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 95px; white-space: nowrap; word-wrap: normal; text- align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">pix_syslog_routerStorm spout[Not supported by viewer]http://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0 ); line-height: 1.2; vertical-align: top; width: 100px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">pix_syslog_sensorKafka topicpix_syslog_sensorbrKafka topichttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 64px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w 3.org/1999/xhtml" style="display:inline-block;text-align:inherit;text-decoration:inherit;">cisco-5-304Kafka topic[Not supported by viewer]http://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0,
metron git commit: METRON-1673 Fix Javadoc errors (justinleet) closes apache/metron#1107
Repository: metron
Updated Branches:
refs/heads/master 0c20fd1a3 -> 2a1cbbbf2
METRON-1673 Fix Javadoc errors (justinleet) closes apache/metron#1107
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2a1cbbbf
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2a1cbbbf
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2a1cbbbf
Branch: refs/heads/master
Commit: 2a1cbbbf2029faec0b265e83f270a31e75ef9d9a
Parents: 0c20fd1
Author: justinleet
Authored: Tue Jul 17 09:18:08 2018 -0400
Committer: leet
Committed: Tue Jul 17 09:18:08 2018 -0400
--
.../metron/profiler/client/window/Window.java | 8 ++---
.../profiler/client/window/WindowProcessor.java | 35
.../client/window/predicates/DayPredicates.java | 2 +-
.../profiler/DefaultMessageDistributor.java | 18 +-
.../apache/metron/rest/config/KafkaConfig.java | 4 +--
.../configuration/IndexingConfigurations.java | 2 +-
.../configuration/SensorParserConfig.java | 6 ++--
.../writer/ConfigurationsStrategies.java| 2 +-
.../common/performance/PerformanceLogger.java | 10 +++---
.../apache/metron/common/system/FakeClock.java | 18 +-
.../metron/common/utils/ReflectionUtils.java| 2 +-
.../configurations/ConfigurationsUpdater.java | 1 -
.../indexing/dao/metaalert/MetaAlertDao.java| 6 ++--
.../metron/writer/BulkWriterComponent.java | 24 ++
.../stellar/common/CachingStellarProcessor.java | 2 +-
.../org/apache/metron/stellar/dsl/Context.java | 2 +-
16 files changed, 72 insertions(+), 70 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/2a1cbbbf/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/Window.java
--
diff --git
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/Window.java
b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/Window.java
index 964d594..9fd6339 100644
---
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/Window.java
+++
b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/Window.java
@@ -20,14 +20,12 @@
package org.apache.metron.profiler.client.window;
import com.google.common.collect.Iterables;
-import org.apache.commons.lang3.Range;
-
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Predicate;
-import java.util.stream.Stream;
+import org.apache.commons.lang3.Range;
/**
* A window is intended to compute the set of window intervals across time
based on a reference time.
@@ -72,7 +70,7 @@ public class Window {
* Get the set of inclusion predicates. If any of these are true as applied
to the window interval start time,
* then a field is included unless it's explicitly excluded.
* @param now
- * @return Iterable> returns a set of inclusion predicates.
If any of these are true as
+ * @return {@code Iterable>} returns a set of inclusion
predicates. If any of these are true as
* applied to the window interval start time, then a field is included
unless it's explicitly excluded
*/
public Iterable> getIncludes(long now) {
@@ -87,7 +85,7 @@ public class Window {
* Get the set of exclusion predicates. If any of these exclusion
predicates are true as applied to the window
* interval start time, then the interval is excluded. NOTE: Exclusions
trump inclusions.
* @param now
- * @return Iterable> returns the set of exclusion
predicates. If any of these exclusion
+ * @return {@code Iterable>} returns the set of exclusion
predicates. If any of these exclusion
* predicates are true as applied to the window interval start time, then
the interval is excluded.
* Exclusions trump inclusions.
*/
http://git-wip-us.apache.org/repos/asf/metron/blob/2a1cbbbf/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java
--
diff --git
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java
b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java
index da4ce3c..66a4f7d 100644
---
a/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/WindowProcessor.java
+++
b/metron-analytics/metron-profiler-client/src/main/java/org/apache/metron/profiler/client/window/Wi
[39/53] [abbrv] metron git commit: METRON-1587 Make collection utility work for HDP search (merrimanr) closes apache/metron#1043
METRON-1587 Make collection utility work for HDP search (merrimanr) closes
apache/metron#1043
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f241f87f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f241f87f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f241f87f
Branch: refs/heads/master
Commit: f241f87f87523f2c6f2203b3f316cff0fd3f20b4
Parents: aed4ffc
Author: merrimanr
Authored: Mon Jun 11 10:09:16 2018 -0500
Committer: merrimanr
Committed: Mon Jun 11 10:09:16 2018 -0500
--
.../package/scripts/indexing_commands.py| 63 +---
.../CURRENT/package/scripts/indexing_master.py | 31 +-
.../package/scripts/params/params_linux.py | 5 ++
metron-platform/metron-solr/README.md | 33 +-
.../src/main/scripts/create_collection.sh | 21 +--
.../src/main/scripts/delete_collection.sh | 18 --
6 files changed, 120 insertions(+), 51 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index eeb2127..69e980b 100755
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -25,6 +25,7 @@ from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute, File
from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.functions.format import format
import metron_service
import metron_security
@@ -85,14 +86,13 @@ class IndexingCommands:
:return: Dict where key is the name of a collection and the
value is a path to file containing the schema definition.
"""
-from params import params
-return {
-"bro": params.bro_schema_path,
-"yaf": params.yaf_schema_path,
-"snort": params.snort_schema_path,
-"error": params.error_schema_path,
-"metaalert": params.meta_schema_path
-}
+return [
+"bro",
+"yaf",
+"snort",
+"error",
+"metaalert"
+]
def is_configured(self):
return self.__configured
@@ -200,6 +200,53 @@ class IndexingCommands:
user=self.__params.metron_user,
err_msg=err_msg.format(template_name))
+def solr_schema_install(self, env):
+from params import params
+env.set_params(params)
+Logger.info("Installing Solr schemas")
+if self.__params.security_enabled:
+metron_security.kinit(self.__params.kinit_path_local,
+ self.__params.solr_keytab_path,
+ self.__params.solr_principal_name,
+ self.__params.solr_user)
+
+commands = IndexingCommands(params)
+for collection_name in commands.get_solr_schemas():
+
+# install the schema
+cmd = format((
+"export ZOOKEEPER={solr_zookeeper_url};"
+"export SECURITY_ENABLED={security_enabled};"
+))
+cmd += "{0}/bin/create_collection.sh {1};"
+
+Execute(
+cmd.format(params.metron_home, collection_name),
+user=self.__params.solr_user)
+
+def solr_schema_delete(self, env):
+from params import params
+env.set_params(params)
+Logger.info("Deleting Solr schemas")
+if self.__params.security_enabled:
+metron_security.kinit(self.__params.kinit_path_local,
+ self.__params.solr_keytab_path,
+ self.__params.solr_principal_name,
+ self.__params.solr_user)
+
+commands = IndexingCommands(params)
+for collection_name in commands.get_solr_schemas():
+# delete the schema
+cmd = format((
+"export ZOOKEEPER={solr_zookeeper_url};"
+"export SECURITY_ENABLED={security_enabled};"
+))
+
[36/53] [abbrv] metron git commit: METRON-1547 Solr Comment Fields (justinleet) closes apache/metron#1037
METRON-1547 Solr Comment Fields (justinleet) closes apache/metron#1037
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a68d031b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a68d031b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a68d031b
Branch: refs/heads/master
Commit: a68d031b01fe677d84abb0d25aca4f2ceaf90c53
Parents: 9348c60
Author: justinleet
Authored: Tue Jun 5 14:59:29 2018 -0400
Committer: leet
Committed: Tue Jun 5 14:59:29 2018 -0400
--
.../alert-details/alert-details.component.ts| 25 ++-
.../src/app/model/comment-add-remove-request.ts | 25 +++
.../src/app/service/update.service.ts | 27 +++
.../rest/controller/UpdateController.java | 23 +++
.../metron/rest/service/UpdateService.java | 3 +
.../rest/service/impl/UpdateServiceImpl.java| 19 +++
.../UpdateControllerIntegrationTest.java| 92 --
.../elasticsearch/dao/ElasticsearchDao.java | 21 +++
.../dao/ElasticsearchMetaAlertDao.java | 22 +++
.../dao/ElasticsearchMetaAlertUpdateDao.java| 23 +++
.../dao/ElasticsearchUpdateDao.java | 70
.../dao/ElasticsearchMetaAlertDaoTest.java | 17 ++
.../ElasticsearchUpdateIntegrationTest.java | 3 +-
.../apache/metron/indexing/dao/HBaseDao.java| 93 +-
.../apache/metron/indexing/dao/IndexDao.java| 2 +
.../metron/indexing/dao/MultiIndexDao.java | 47 ++
.../indexing/dao/search/AlertComment.java | 130 ++
.../dao/update/CommentAddRemoveRequest.java | 78 +
.../metron/indexing/dao/update/Document.java| 14 +-
.../metron/indexing/dao/update/PatchUtil.java | 50 --
.../metron/indexing/dao/update/UpdateDao.java | 33 +++-
.../apache/metron/indexing/dao/InMemoryDao.java | 17 ++
.../indexing/dao/InMemoryMetaAlertDao.java | 17 ++
.../indexing/dao/UpdateIntegrationTest.java | 169 ++-
.../AbstractLuceneMetaAlertUpdateDaoTest.java | 17 ++
.../integration/HBaseDaoIntegrationTest.java| 79 -
.../src/main/config/schema/bro/schema.xml | 3 +
.../src/main/config/schema/snort/schema.xml | 3 +
.../src/main/config/schema/yaf/schema.xml | 3 +
.../org/apache/metron/solr/dao/SolrDao.java | 41 -
.../metron/solr/dao/SolrMetaAlertDao.java | 21 +++
.../metron/solr/dao/SolrMetaAlertUpdateDao.java | 23 +++
.../metron/solr/dao/SolrRetrieveLatestDao.java | 1 +
.../apache/metron/solr/dao/SolrSearchDao.java | 17 ++
.../apache/metron/solr/dao/SolrUpdateDao.java | 114 -
.../apache/metron/solr/dao/SolrUtilities.java | 36 +++-
.../org/apache/metron/solr/dao/SolrDaoTest.java | 5 +-
.../metron/solr/dao/SolrMetaAlertDaoTest.java | 18 +-
.../metron/solr/dao/SolrUpdateDaoTest.java | 94 +--
.../integration/SolrSearchIntegrationTest.java | 6 +-
.../integration/SolrUpdateIntegrationTest.java | 24 ++-
.../resources/config/test/conf/managed-schema | 3 +
42 files changed, 1400 insertions(+), 128 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/a68d031b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index e1c1685..6a07e08 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -30,6 +30,7 @@ import {AlertComment} from './alert-comment';
import {AuthenticationService} from '../../service/authentication.service';
import {MetronDialogBox} from '../../shared/metron-dialog-box';
import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from
'../../utils/constants';
+import {CommentAddRemoveRequest} from "../../model/comment-add-remove-request";
export enum AlertState {
NEW, OPEN, ESCALATE, DISMISS, RESOLVE
@@ -204,10 +205,15 @@ export class AlertDetailsComponent implements OnInit {
}
onAddComment() {
-let alertComment = new AlertComment(this.alertCommentStr,
this.authenticationService.getCurrentUserName(), new Date().getTime());
-let tAlertComments = this.alertCommentsWrapper.map(alertsWrapper =>
alertsWrapper.alertComment);
-tAlertComments.unshift(alertComment);
-this.patchAlert(new Patch('add', '/comments', tAlertComments));
+let commentRequest = new CommentAddRemoveRequest();
+commentRequest.guid = this.alertSource.guid;
+commentRequest.comment = this.alertCommentStr;
+commentReq
[32/53] [abbrv] metron git commit: METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970
METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/49f851e0 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/49f851e0 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/49f851e0 Branch: refs/heads/master Commit: 49f851e0b8c2ffa1cdd7c8f169bed3dfa07cf35c Parents: eb33666 Author: justinleet Authored: Wed May 23 10:32:34 2018 -0400 Committer: leet Committed: Wed May 23 10:32:34 2018 -0400 -- metron-analytics/metron-profiler/.gitignore |1 + .../apache/metron/rest/config/IndexConfig.java | 12 +- .../rest/service/impl/MetaAlertServiceImpl.java |5 +- .../rest/service/impl/SearchServiceImpl.java|2 +- .../rest/service/impl/UpdateServiceImpl.java|2 +- .../MetaAlertControllerIntegrationTest.java |6 +- .../UpdateControllerIntegrationTest.java|4 +- .../elasticsearch/dao/ElasticsearchDao.java | 65 +- .../dao/ElasticsearchMetaAlertDao.java | 641 ++- ...ElasticsearchMetaAlertRetrieveLatestDao.java | 44 + .../dao/ElasticsearchMetaAlertSearchDao.java| 110 ++ .../dao/ElasticsearchMetaAlertUpdateDao.java| 219 .../dao/ElasticsearchRetrieveLatestDao.java | 151 +++ .../dao/ElasticsearchSearchDao.java | 102 -- .../dao/ElasticsearchUpdateDao.java | 10 +- .../elasticsearch/utils/ElasticsearchUtils.java | 64 ++ .../elasticsearch/dao/ElasticsearchDaoTest.java | 70 +- .../dao/ElasticsearchMetaAlertDaoTest.java | 164 +-- .../ElasticsearchMetaAlertIntegrationTest.java | 986 ++--- .../ElasticsearchSearchIntegrationTest.java | 64 +- .../ElasticsearchUpdateIntegrationTest.java | 84 +- .../components/ElasticSearchComponent.java | 26 +- metron-platform/metron-indexing/README.md |2 +- metron-platform/metron-indexing/pom.xml |8 +- .../metron/indexing/dao/AccessConfig.java | 10 + .../apache/metron/indexing/dao/IndexDao.java| 141 +-- .../metron/indexing/dao/MetaAlertDao.java | 154 --- .../metron/indexing/dao/RetrieveLatestDao.java | 67 ++ .../metaalert/DeferredMetaAlertIndexDao.java| 42 + .../metaalert/MetaAlertAddRemoveRequest.java|1 - .../indexing/dao/metaalert/MetaAlertConfig.java | 74 ++ .../dao/metaalert/MetaAlertConstants.java | 30 + .../indexing/dao/metaalert/MetaAlertDao.java| 77 ++ .../metaalert/MetaAlertRetrieveLatestDao.java | 25 + .../dao/metaalert/MetaAlertSearchDao.java | 35 + .../dao/metaalert/MetaAlertUpdateDao.java | 146 +++ .../indexing/dao/metaalert/MetaScores.java | 52 +- .../AbstractLuceneMetaAlertUpdateDao.java | 334 ++ .../metron/indexing/dao/search/SearchDao.java | 22 +- .../indexing/dao/search/SearchResponse.java | 10 +- .../metron/indexing/dao/update/PatchUtil.java | 50 + .../metron/indexing/dao/update/UpdateDao.java | 47 + .../metron/indexing/util/IndexingCacheUtil.java | 35 + .../indexing/dao/InMemoryMetaAlertDao.java | 69 +- .../indexing/dao/SearchIntegrationTest.java | 60 +- .../indexing/dao/UpdateIntegrationTest.java | 87 +- .../dao/metaalert/MetaAlertIntegrationTest.java | 1012 ++ .../indexing/dao/metaalert/MetaScoresTest.java | 75 ++ .../AbstractLuceneMetaAlertUpdateDaoTest.java | 854 +++ .../integration/IndexingIntegrationTest.java|4 +- metron-platform/metron-pcap-backend/.gitignore |1 + metron-platform/metron-solr/pom.xml |4 +- .../src/main/config/schema/bro/schema.xml |3 + .../src/main/config/schema/metaalert/schema.xml | 39 +- .../src/main/config/schema/snort/schema.xml |3 + .../src/main/config/schema/yaf/schema.xml |3 + .../org/apache/metron/solr/dao/SolrDao.java | 37 +- .../metron/solr/dao/SolrMetaAlertDao.java | 285 +++-- .../dao/SolrMetaAlertRetrieveLatestDao.java | 77 ++ .../metron/solr/dao/SolrMetaAlertSearchDao.java | 211 .../metron/solr/dao/SolrMetaAlertUpdateDao.java | 216 .../metron/solr/dao/SolrRetrieveLatestDao.java | 81 ++ .../apache/metron/solr/dao/SolrSearchDao.java | 127 +-- .../apache/metron/solr/dao/SolrUpdateDao.java | 51 +- .../apache/metron/solr/dao/SolrUtilities.java | 92 ++ .../org/apache/metron/solr/dao/SolrDaoTest.java | 61 +- .../metron/solr/dao/SolrMetaAlertDaoTest.java | 137 +++ .../metron/solr/dao/SolrSearchDaoTest.java | 176 ++- .../metron/solr/dao/SolrUpdateDaoTest.java | 19 +- .../metron/solr/dao/SolrUtilitiesTest.java | 48 + .../SolrIndexingIntegrationTest.java|5 +- .../SolrMetaAlertIntegrationTest.java | 397 +++ .../integration/SolrSearchIntegrationTest.java | 59
[42/53] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e7233b0f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e7233b0f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e7233b0f
Branch: refs/heads/master
Commit: e7233b0f82f372e6b1339de56a115fa7e4581f07
Parents: 4170887
Author: cstella
Authored: Mon Jun 11 21:41:06 2018 -0400
Committer: cstella
Committed: Tue Jun 12 09:46:46 2018 -0400
--
.../org/apache/metron/common/Constants.java | 1 +
.../elasticsearch/dao/ElasticsearchDao.java | 4 ++
.../dao/ElasticsearchMetaAlertDao.java | 30 +-
.../dao/ElasticsearchMetaAlertUpdateDao.java| 4 +-
.../indexing/dao/metaalert/MetaAlertConfig.java | 43
.../AbstractLuceneMetaAlertUpdateDao.java | 3 ++
.../dao/metaalert/MetaAlertIntegrationTest.java | 2 +-
.../AbstractLuceneMetaAlertUpdateDaoTest.java | 28 +
.../metron/solr/dao/SolrMetaAlertDao.java | 29 +
.../metron/solr/dao/SolrMetaAlertSearchDao.java | 15 ---
.../metron/solr/dao/SolrMetaAlertUpdateDao.java | 2 +-
.../SolrMetaAlertIntegrationTest.java | 28 +
12 files changed, 129 insertions(+), 60 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
--
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
index f74660c..4a8bea2 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java
@@ -28,6 +28,7 @@ public class Constants {
public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000;
public static final String SENSOR_TYPE = "source.type";
public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field";
+ public static final String THREAT_SCORE_FIELD_PROPERTY =
"threat.triage.score.field";
public static final String ENRICHMENT_TOPIC = "enrichments";
public static final String INDEXING_TOPIC = "indexing";
public static final String ERROR_STREAM = "error";
http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
index eae0a39..3eb86ce 100644
---
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
+++
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java
@@ -85,6 +85,10 @@ public class ElasticsearchDao implements IndexDao {
//uninitialized.
}
+ public AccessConfig getAccessConfig() {
+return accessConfig;
+ }
+
@Override
public synchronized void init(AccessConfig config) {
if (this.client == null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/e7233b0f/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index f73a640..649077e 100644
---
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -68,6 +68,7 @@ import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
import static org.apache.metron.common.Constants.GUID;
@@ -83,10 +84,8 @@ public class ElasticsearchMetaAlertDao implements
MetaAlertDao {
public static final String THREAT_TRIAGE_FIELD =
MetaAlertConstants.THREAT_FIELD_DEFAULT
.replace('.', ':');
public static final String METAALERTS_INDEX = "metaalert_index";
-
public static final String SOURCE_TYPE_FIELD =
[37/53] [abbrv] metron git commit: METRON-1603: Fix multivalue field errors in Bro Solr schema (mmiklavc via mmiklavc) closes apache/metron#1051
METRON-1603: Fix multivalue field errors in Bro Solr schema (mmiklavc via mmiklavc) closes apache/metron#1051 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/95e65284 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/95e65284 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/95e65284 Branch: refs/heads/master Commit: 95e65284980bbe5ce07cd2decb6ce66023c5c84f Parents: a68d031 Author: mmiklavc Authored: Wed Jun 6 13:36:46 2018 -0600 Committer: Michael Miklavcic Committed: Wed Jun 6 13:36:46 2018 -0600 -- .../src/main/config/schema/bro/schema.xml | 24 ++-- .../src/test/resources/example_data/bro | 8 +++ 2 files changed, 20 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/95e65284/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml -- diff --git a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml index 1326dfc..ea9f6d3 100644 --- a/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml +++ b/metron-platform/metron-solr/src/main/config/schema/bro/schema.xml @@ -146,8 +146,8 @@ - - + + - + - + @@ -311,9 +311,9 @@ - + - +
[11/53] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1006d9d6 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1006d9d6 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1006d9d6 Branch: refs/heads/master Commit: 1006d9d6e473d56d5ada2e7ff3011b98f047ec7c Parents: 8cc8aab b48ab93 Author: Michael Miklavcic Authored: Tue Feb 27 14:43:31 2018 -0700 Committer: Michael Miklavcic Committed: Tue Feb 27 14:43:31 2018 -0700 -- Upgrading.md| 18 + metron-deployment/amazon-ec2/README.md | 25 +- .../metron/rest/controller/KafkaController.java | 1 + .../metron/rest/service/KafkaService.java | 9 + .../rest/service/impl/KafkaServiceImpl.java | 33 +- .../rest/service/impl/KafkaServiceImplTest.java | 10 + .../apache/metron/common/utils/JSONUtils.java | 7 + .../dao/ElasticsearchMetaAlertDao.java | 15 +- .../apache/metron/indexing/dao/IndexDao.java| 23 +- .../indexing/dao/update/PatchRequest.java | 7 +- .../paloalto/BasicPaloAltoFirewallParser.java | 333 + .../BasicPaloAltoFirewallParserTest.java| 493 ++- .../logData/PaloAltoFirewallParserTest.txt | 2 - site/_includes/footer.html | 3 +- site/community/index.md | 4 + site/index.html | 30 ++ 16 files changed, 853 insertions(+), 160 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/1006d9d6/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java --
[43/53] [abbrv] metron git commit: Merge branch 'feature/METRON-1416-upgrade-solr' of https://git-wip-us.apache.org/repos/asf/metron into feature/METRON-1416-upgrade-solr
Merge branch 'feature/METRON-1416-upgrade-solr' of https://git-wip-us.apache.org/repos/asf/metron into feature/METRON-1416-upgrade-solr Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/47685619 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/47685619 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/47685619 Branch: refs/heads/master Commit: 4768561921b01eb11c14149ed9f8a07a7b0149f0 Parents: e7233b0 0717cfc Author: cstella Authored: Tue Jun 12 09:47:55 2018 -0400 Committer: cstella Committed: Tue Jun 12 09:47:55 2018 -0400 -- --
[34/53] [abbrv] metron git commit: METRON-1589 '/api/v1/search/search' fails when 'Solr Zookeeper Urls' has comma separated multiple zookeeper urls (justinleet) closes apache/metron#1040
METRON-1589 '/api/v1/search/search' fails when 'Solr Zookeeper Urls' has comma
separated multiple zookeeper urls (justinleet) closes apache/metron#1040
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9348c608
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9348c608
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9348c608
Branch: refs/heads/master
Commit: 9348c608175afd311177b82fe31737f2f984c175
Parents: 7a071f6
Author: justinleet
Authored: Tue Jun 5 07:44:20 2018 -0400
Committer: leet
Committed: Tue Jun 5 07:44:20 2018 -0400
--
.../org/apache/metron/solr/SolrConstants.java | 1 +
.../metron/solr/dao/SolrColumnMetadataDao.java | 14 +++---
.../org/apache/metron/solr/dao/SolrDao.java | 35 ++
.../metron/solr/dao/SolrMetaAlertDao.java | 2 +-
.../dao/SolrMetaAlertRetrieveLatestDao.java | 2 +-
.../metron/solr/dao/SolrMetaAlertUpdateDao.java | 2 +-
.../apache/metron/solr/writer/SolrWriter.java | 43 +-
.../metron/solr/dao/SolrColumnMetadataTest.java | 11 ++---
.../org/apache/metron/solr/dao/SolrDaoTest.java | 48 ++--
.../metron/solr/dao/SolrMetaAlertDaoTest.java | 4 +-
.../metron/solr/dao/SolrUpdateDaoTest.java | 3 +-
.../SolrIndexingIntegrationTest.java| 4 +-
.../SolrMetaAlertIntegrationTest.java | 5 +-
.../integration/SolrSearchIntegrationTest.java | 4 +-
.../integration/SolrUpdateIntegrationTest.java | 3 +-
.../schema/SchemaValidationIntegrationTest.java | 3 +-
16 files changed, 126 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/9348c608/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
--
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
index 879b983..56f1413 100644
---
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
+++
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/SolrConstants.java
@@ -27,4 +27,5 @@ public class SolrConstants {
public static final String REQUEST_COLLECTIONS_PATH = "/admin/collections";
public static final String RESPONSE_COLLECTIONS = "collections";
public static final String SOLR_WRITER_NAME = "solr";
+ public static final String SOLR_ZOOKEEPER = "solr.zookeeper";
}
http://git-wip-us.apache.org/repos/asf/metron/blob/9348c608/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
--
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
index 1e074a9..22c6efa 100644
---
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
+++
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrColumnMetadataDao.java
@@ -29,6 +29,7 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.metron.indexing.dao.ColumnMetadataDao;
import org.apache.metron.indexing.dao.search.FieldType;
+import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -61,10 +62,10 @@ public class SolrColumnMetadataDao implements
ColumnMetadataDao {
solrTypeMap = Collections.unmodifiableMap(fieldTypeMap);
}
- private String zkHost;
+ private transient SolrClient client;
- public SolrColumnMetadataDao(String zkHost) {
-this.zkHost = zkHost;
+ public SolrColumnMetadataDao(SolrClient client) {
+this.client = client;
}
@Override
@@ -115,14 +116,11 @@ public class SolrColumnMetadataDao implements
ColumnMetadataDao {
protected List> getIndexFields(String index)
throws IOException, SolrServerException {
-CloudSolrClient client = new
CloudSolrClient.Builder().withZkHost(zkHost).build();
-client.setDefaultCollection(index);
-
List> indexFields = new ArrayList<>();
// Get all the fields in use, including dynamic fields
LukeRequest lukeRequest = new LukeRequest();
-LukeResponse lukeResponse = lukeRequest.process(client);
+LukeResponse lukeResponse = lukeRequest.process(client, index);
for (Entry field :
lukeResponse.getFieldInfo().entrySet()) {
Map fieldData = new HashMap<>();
fieldData.put("
[47/53] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a89a72c3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a89a72c3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a89a72c3
Branch: refs/heads/master
Commit: a89a72c3b2ceb5066fd7a310636507ce274c66ad
Parents: 10271c4 b081e80
Author: cstella
Authored: Fri Jun 15 20:32:18 2018 -0400
Committer: cstella
Committed: Fri Jun 15 20:32:18 2018 -0400
--
.../alert-details/alert-details.component.html | 4 +--
.../alert-details/alert-details.component.ts| 24 ++---
.../alerts-list/alerts-list.component.html | 4 +--
.../alerts/alerts-list/alerts-list.component.ts | 13 ++
.../src/app/alerts/alerts-list/query-builder.ts | 5 ++--
.../table-view/table-view.component.html| 18 ++---
.../table-view/table-view.component.ts | 19 --
.../tree-view/tree-view.component.html | 16 ++--
.../tree-view/tree-view.component.ts| 27 +---
.../meta-alerts/meta-alerts.component.html | 2 +-
.../alerts/meta-alerts/meta-alerts.component.ts | 11 +---
.../src/app/model/group-request.ts | 2 +-
.../src/app/service/global-config.service.ts| 16 +---
.../metron-alerts/src/app/utils/constants.ts| 3 +--
.../ElasticsearchMetaAlertIntegrationTest.java | 5 ++--
15 files changed, 120 insertions(+), 49 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
--
diff --cc
metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index ffd7dd0,c939f04..39d47ac
---
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@@ -29,8 -30,8 +30,9 @@@ import {Patch} from '../../model/patch'
import {AlertComment} from './alert-comment';
import {AuthenticationService} from '../../service/authentication.service';
import {MetronDialogBox} from '../../shared/metron-dialog-box';
- import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
-import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from
'../../utils/constants';
+import {CommentAddRemoveRequest} from
"../../model/comment-add-remove-request";
++import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
+ import { GlobalConfigService } from '../../service/global-config.service';
export enum AlertState {
NEW, OPEN, ESCALATE, DISMISS, RESOLVE
http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
--
diff --cc
metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
index 98459db,4496e37..07f7015
---
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
@@@ -36,10 -36,10 +36,10 @@@ import {AlertSearchDirective} from '../
import {SearchResponse} from '../../model/search-response';
import {ElasticsearchUtils} from '../../utils/elasticsearch-utils';
import {Filter} from '../../model/filter';
- import {THREAT_SCORE_FIELD_NAME, TIMESTAMP_FIELD_NAME, ALL_TIME} from
'../../utils/constants';
+ import {TIMESTAMP_FIELD_NAME, ALL_TIME} from '../../utils/constants';
import {TableViewComponent} from './table-view/table-view.component';
import {Pagination} from '../../model/pagination';
-import {META_ALERTS_SENSOR_TYPE, META_ALERTS_INDEX} from
'../../utils/constants';
+import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
import {MetaAlertService} from '../../service/meta-alert.service';
import {Facets} from '../../model/facets';
import { GlobalConfigService } from '../../service/global-config.service';
http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.html
--
http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/alerts/alerts-list/table-view/table-view.component.ts
--
http://git-wip-us.apache.org/repos/asf/metron/blob/a89a72c3/metron-interface/metron-alerts/src/app/utils/constants.ts
--
[38/53] [abbrv] metron git commit: METRON-1585 SolrRetrieveLatestDao does not use the collection lookup (justinleet via merrimanr) closes apache/metron#1050
METRON-1585 SolrRetrieveLatestDao does not use the collection lookup
(justinleet via merrimanr) closes apache/metron#1050
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/aed4ffcb
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/aed4ffcb
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/aed4ffcb
Branch: refs/heads/master
Commit: aed4ffcb24cf9a95c1347cccd5dd79fce7c77a6b
Parents: 95e6528
Author: justinleet
Authored: Fri Jun 8 12:16:59 2018 -0500
Committer: merrimanr
Committed: Fri Jun 8 12:16:59 2018 -0500
--
.../org/apache/metron/solr/dao/SolrDao.java | 10 +-
.../metron/solr/dao/SolrRetrieveLatestDao.java | 37 +++-
.../apache/metron/solr/dao/SolrSearchDao.java | 17 --
.../apache/metron/solr/dao/SolrUpdateDao.java | 18 +-
.../apache/metron/solr/dao/SolrUtilities.java | 18 ++
.../org/apache/metron/solr/dao/SolrDaoTest.java | 4 +-
.../metron/solr/dao/SolrSearchDaoTest.java | 2 +-
.../metron/solr/dao/SolrUpdateDaoTest.java | 4 +-
.../SolrRetrieveLatestIntegrationTest.java | 207 +++
.../integration/SolrUpdateIntegrationTest.java | 1 +
10 files changed, 266 insertions(+), 52 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
--
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
index 7db0ab5..a840bb4 100644
---
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
+++
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
@@ -22,16 +22,13 @@ import static
org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
import com.google.common.base.Splitter;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.function.Function;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.ColumnMetadataDao;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
-import org.apache.metron.indexing.dao.search.AlertComment;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.GroupRequest;
@@ -43,7 +40,6 @@ import
org.apache.metron.indexing.dao.update.CommentAddRemoveRequest;
import org.apache.metron.indexing.dao.update.Document;
import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
import org.apache.metron.indexing.dao.update.PatchRequest;
-import org.apache.metron.solr.SolrConstants;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
@@ -92,7 +88,7 @@ public class SolrDao implements IndexDao {
this.accessConfig = config;
this.client = getSolrClient(getZkHosts());
this.solrSearchDao = new SolrSearchDao(this.client, this.accessConfig);
- this.solrRetrieveLatestDao = new SolrRetrieveLatestDao(this.client);
+ this.solrRetrieveLatestDao = new SolrRetrieveLatestDao(this.client,
this.accessConfig);
this.solrUpdateDao = new SolrUpdateDao(this.client,
this.solrRetrieveLatestDao, this.accessConfig);
this.solrColumnMetadataDao = new SolrColumnMetadataDao(this.client);
}
@@ -118,8 +114,8 @@ public class SolrDao implements IndexDao {
}
@Override
- public Document getLatest(String guid, String collection) throws IOException
{
-return this.solrRetrieveLatestDao.getLatest(guid, collection);
+ public Document getLatest(String guid, String sensorType) throws IOException
{
+return this.solrRetrieveLatestDao.getLatest(guid, sensorType);
}
@Override
http://git-wip-us.apache.org/repos/asf/metron/blob/aed4ffcb/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
--
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
index b3bc564..40262ba 100644
---
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
+++
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrRetrieveLatestDao.java
@@ -19,13 +19,16 @@
package
[31/53] [abbrv] metron git commit: METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
index 24f7a27..98dc66d 100644
---
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
+++
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/utils/ElasticsearchUtils.java
@@ -36,18 +36,24 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.utils.HDFSUtils;
import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.netty.utils.NettyRuntimeWrapper;
import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.codehaus.jackson.map.ObjectMapper;
+import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -321,4 +327,62 @@ public class ElasticsearchUtils {
return json;
}
+
+ /**
+ * Elasticsearch queries default to 10 records returned. Some internal
queries require that all
+ * results are returned. Rather than setting an arbitrarily high size, this
method pages through results
+ * and returns them all in a single SearchResponse.
+ * @param qb A QueryBuilder that provides the query to be run.
+ * @return A SearchResponse containing the appropriate results.
+ */
+ public static SearchResponse queryAllResults(TransportClient
transportClient,
+ QueryBuilder qb,
+ String index,
+ int pageSize
+ ) {
+SearchRequestBuilder searchRequestBuilder = transportClient
+.prepareSearch(index)
+.addStoredField("*")
+.setFetchSource(true)
+.setQuery(qb)
+.setSize(pageSize);
+org.elasticsearch.action.search.SearchResponse esResponse =
searchRequestBuilder
+.execute()
+.actionGet();
+List allResults = getSearchResults(esResponse);
+long total = esResponse.getHits().getTotalHits();
+if (total > pageSize) {
+ int pages = (int) (total / pageSize) + 1;
+ for (int i = 1; i < pages; i++) {
+int from = i * pageSize;
+searchRequestBuilder.setFrom(from);
+esResponse = searchRequestBuilder
+.execute()
+.actionGet();
+allResults.addAll(getSearchResults(esResponse));
+ }
+}
+SearchResponse searchResponse = new SearchResponse();
+searchResponse.setTotal(total);
+searchResponse.setResults(allResults);
+return searchResponse;
+ }
+
+ /**
+ * Transforms a list of Elasticsearch SearchHits to a list of SearchResults
+ * @param searchResponse An Elasticsearch SearchHit to be converted.
+ * @return The list of SearchResults for the SearchHit
+ */
+ protected static List getSearchResults(
+ org.elasticsearch.action.search.SearchResponse searchResponse) {
+return Arrays.stream(searchResponse.getHits().getHits()).map(searchHit -> {
+ SearchResult searchResult = new SearchResult();
+ searchResult.setId(searchHit.getId());
+ searchResult.setSource(searchHit.getSource());
+ searchResult.setScore(searchHit.getScore());
+ searchResult.setIndex(searchHit.getIndex());
+ return searchResult;
+}
+).collect(Collectors.toList());
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchDaoTest.java
index ca1b860..6c3c327 100644
---
[30/53] [abbrv] metron git commit: METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
--
diff --git
a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
index fe546bd..4187428 100644
---
a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
+++
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
@@ -17,151 +17,18 @@
*/
package org.apache.metron.indexing.dao;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import org.apache.metron.common.utils.JSONUtils;
-import org.apache.metron.indexing.dao.search.FieldType;
-import org.apache.metron.indexing.dao.search.GetRequest;
-import org.apache.metron.indexing.dao.search.GroupRequest;
-import org.apache.metron.indexing.dao.search.GroupResponse;
-import org.apache.metron.indexing.dao.search.InvalidSearchException;
-import org.apache.metron.indexing.dao.search.SearchRequest;
-import org.apache.metron.indexing.dao.search.SearchResponse;
-import org.apache.metron.indexing.dao.update.Document;
-import org.apache.metron.indexing.dao.update.OriginalNotFoundException;
-import org.apache.metron.indexing.dao.update.PatchRequest;
-import org.apache.metron.indexing.dao.update.ReplaceRequest;
+import org.apache.metron.indexing.dao.search.SearchDao;
+import org.apache.metron.indexing.dao.update.UpdateDao;
/**
* The IndexDao provides a common interface for retrieving and storing data in
a variety of persistent stores.
* Document reads and writes require a GUID and sensor type with an index
being optional.
*/
-public interface IndexDao {
-
- /**
- * Return search response based on the search request
- *
- * @param searchRequest
- * @return
- * @throws InvalidSearchException
- */
- SearchResponse search(SearchRequest searchRequest) throws
InvalidSearchException;
-
- GroupResponse group(GroupRequest groupRequest) throws InvalidSearchException;
+public interface IndexDao extends UpdateDao, SearchDao, RetrieveLatestDao,
ColumnMetadataDao {
/**
* Initialize the DAO with the AccessConfig object.
- * @param config
+ * @param config The config to use for initialization
*/
void init(AccessConfig config);
-
- /**
- * Return the latest version of a document given the GUID and the sensor
type.
- *
- * @param guid The GUID for the document
- * @param sensorType The sensor type of the document
- * @return The Document matching or null if not available.
- * @throws IOException
- */
- Document getLatest(String guid, String sensorType) throws IOException;
-
- /**
- * Return a list of the latest versions of documents given a list of GUIDs
and sensor types.
- *
- * @param getRequests A list of get requests for documents
- * @return A list of documents matching or an empty list in not available.
- * @throws IOException
- */
- Iterable getAllLatest(List getRequests) throws
IOException;
-
- /**
- * Return the latest version of a document given a GetRequest.
- * @param request The GetRequest which indicates the GUID and sensor type.
- * @return Optionally the document (dependent upon existence in the index).
- * @throws IOException
- */
- default Optional> getLatestResult(GetRequest request)
throws IOException {
-Document ret = getLatest(request.getGuid(), request.getSensorType());
-if(ret == null) {
- return Optional.empty();
-}
-else {
- return Optional.ofNullable(ret.getDocument());
-}
- }
-
- /**
- * Update a given Document and optionally the index where the document
exists. This is a full update,
- * meaning the current document will be replaced if it exists or a new
document will be created if it does
- * not exist. Partial updates are not supported in this method.
- *
- * @param update The document to replace from the index.
- * @param index The index where the document lives.
- * @throws IOException
- */
- void update(Document update, Optional index) throws IOException;
-
- /**
- * Similar to the update method but accepts multiple documents and performs
updates in batch.
- *
- * @param updates A map of the documents to update to the index where they
live.
- * @throws IOException
- */
- void batchUpdate(Map> updates) throws IOException;
-
- /**
- * Update a document in an index given a JSON Patch (see RFC 6902 at
https://tools.ietf.org/html/rfc6902)
- * @param request The patch request
- * @param timestamp Optionally a timestamp to set. If not specified then
current time is used.
- * @throws OriginalNotFoundException If the original is not found, then it
cannot be patched.
- * @throws IOException
- */
- default
[45/53] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0e484279 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0e484279 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0e484279 Branch: refs/heads/master Commit: 0e4842799f8a3afa15074cb121bdbd205e50fb58 Parents: 4c4452e 828ab71 Author: cstella Authored: Thu Jun 14 12:18:43 2018 -0400 Committer: cstella Committed: Thu Jun 14 12:18:43 2018 -0400 -- NOTICE | 6 + dependencies_with_url.csv | 4 + metron-platform/metron-api/pom.xml | 4 - .../parsers/topology/ParserTopologyBuilder.java | 2 +- .../metron-storm-kafka-override/pom.xml | 83 - .../KafkaSpoutRetryExponentialBackoff.java | 336 +++ .../storm/kafka/spout/internal/Timer.java | 9 +- metron-platform/metron-storm-kafka/pom.xml | 5 - pom.xml | 4 +- site/documentation/index.md | 14 +- 10 files changed, 442 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/0e484279/dependencies_with_url.csv -- http://git-wip-us.apache.org/repos/asf/metron/blob/0e484279/pom.xml --
[49/53] [abbrv] metron git commit: METRON-1609 Elasticsearch settings in Ambari should not be required if Solr is the indexer (nickwallen) closes apache/metron#1056
METRON-1609 Elasticsearch settings in Ambari should not be required if Solr is
the indexer (nickwallen) closes apache/metron#1056
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ed4dfb93
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ed4dfb93
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ed4dfb93
Branch: refs/heads/master
Commit: ed4dfb93641c9c9a84b61b5daddbebd962b3b20c
Parents: 2bf6650
Author: nickwallen
Authored: Thu Jun 21 11:28:30 2018 -0400
Committer: nickallen
Committed: Thu Jun 21 11:28:30 2018 -0400
--
.../METRON/CURRENT/configuration/metron-env.xml | 32 --
.../package/scripts/indexing_commands.py| 4 +-
.../CURRENT/package/scripts/indexing_master.py | 10 +-
.../CURRENT/package/scripts/metron_service.py | 101 ++-
4 files changed, 113 insertions(+), 34 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index 0b64f3a..cdef7cf 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -81,30 +81,54 @@
metron
Name of Elasticsearch Cluster
Elasticsearch Cluster Name
+
+true
+
-
+
es_hosts
Comma delimited list of Elasticsearch Master Hosts:
eshost1,eshost2
Elasticsearch Hosts
+
+true
+
es_binary_port
9300
Elasticsearch binary port. (9300)
Elasticsearch Binary Port
+
+true
+
es_http_port
9200
Elasticsearch HTTP port. (9200)
Elasticsearch HTTP port
+
+true
+
+
+
+es_date_format
+Elasticsearch Date Format
+.MM.dd.HH
+Elasticsearch Date Format
+
+true
+
solr_zookeeper_url
{{zookeeper_quorum}}
Comma delimited list of Zookeeper Urls:
zkhost1:2181,zkhost1:2181
Solr Zookeeper Urls
+
+true
+
storm_rest_addr
@@ -137,10 +161,4 @@
true
-
-es_date_format
-Elasticsearch Date Format
-.MM.dd.HH
-Elasticsearch Date Format
-
http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 69e980b..4802add 100755
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -30,7 +30,6 @@ from resource_management.libraries.functions.format import
format
import metron_service
import metron_security
-
# Wrap major operations and functionality in this class
class IndexingCommands:
__params = None
@@ -394,7 +393,6 @@ class IndexingCommands:
is_random_access_running =
topologies[self.__random_access_indexing_topology] in ['ACTIVE', 'REBALANCING']
return is_random_access_running
-
def is_topology_active(self, env):
return self.is_batch_topology_active(env) and
self.is_random_access_topology_active(env)
@@ -403,6 +401,8 @@ class IndexingCommands:
Performs a service check for Indexing.
:param env: Environment
"""
+metron_service.check_indexer_parameters()
+
Logger.info('Checking Kafka topics for Indexing')
metron_service.check_kafka_topics(self.__params, self.__get_topics())
[27/53] [abbrv] metron git commit: METRON-1421 Create a SolrMetaAlertDao (justinleet) closes apache/metron#970
http://git-wip-us.apache.org/repos/asf/metron/blob/49f851e0/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
--
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
index f1b7102..53ac083 100644
---
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
+++
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrSearchDao.java
@@ -18,10 +18,17 @@
package org.apache.metron.solr.dao;
import com.fasterxml.jackson.core.JsonProcessingException;
-import org.apache.metron.common.Constants;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.AccessConfig;
-import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.indexing.dao.search.Group;
import org.apache.metron.indexing.dao.search.GroupOrder;
import org.apache.metron.indexing.dao.search.GroupOrderType;
@@ -35,7 +42,6 @@ import org.apache.metron.indexing.dao.search.SearchResponse;
import org.apache.metron.indexing.dao.search.SearchResult;
import org.apache.metron.indexing.dao.search.SortField;
import org.apache.metron.indexing.dao.search.SortOrder;
-import org.apache.metron.indexing.dao.update.Document;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.ORDER;
@@ -45,25 +51,11 @@ import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-import static org.apache.metron.common.Constants.SENSOR_TYPE;
-
public class SolrSearchDao implements SearchDao {
private static final Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -78,6 +70,13 @@ public class SolrSearchDao implements SearchDao {
@Override
public SearchResponse search(SearchRequest searchRequest) throws
InvalidSearchException {
+return search(searchRequest, null);
+ }
+
+ // Allow for the fieldList to be explicitly specified, letting things like
metaalerts expand on them.
+ // If null, use whatever the searchRequest defines.
+ public SearchResponse search(SearchRequest searchRequest, String fieldList)
+ throws InvalidSearchException {
if (searchRequest.getQuery() == null) {
throw new InvalidSearchException("Search query is invalid: null");
}
@@ -89,7 +88,7 @@ public class SolrSearchDao implements SearchDao {
"Search result size must be less than " +
accessConfig.getMaxSearchResults());
}
try {
- SolrQuery query = buildSearchRequest(searchRequest);
+ SolrQuery query = buildSearchRequest(searchRequest, fieldList);
QueryResponse response = client.query(query);
return buildSearchResponse(searchRequest, response);
} catch (SolrException | IOException | SolrServerException e) {
@@ -108,6 +107,7 @@ public class SolrSearchDao implements SearchDao {
.setStart(0)
.setRows(0)
.setQuery(groupRequest.getQuery());
+
query.set("collection", getCollections(groupRequest.getIndices()));
Optional scoreField = groupRequest.getScoreField();
if (scoreField.isPresent()) {
@@ -125,39 +125,10 @@ public class SolrSearchDao implements SearchDao {
}
}
- @Override
- public Document getLatest(String guid, String collection) throws IOException
{
-try {
- SolrDocument solrDocument = client.getById(collection, guid);
- return toDocument(solrDocument);
-} catch (SolrServerException e) {
- throw new IOException(e);
-}
- }
-
- @Override
- public Iterable getAllLatest(List getRequests) throws
IOException {
-Map> collectionIdMap = new HashMap<>();
-for (GetRequest getRequest: getRequests) {
- Collection ids =
collectionIdMap.getOrDefault(getRequest.getSensorType(), new HashSet<>());
-
[46/53] [abbrv] metron git commit: METRON-1613 Metaalerts status update broken in Alerts UI (merrimanr) closes apache/metron#1059
METRON-1613 Metaalerts status update broken in Alerts UI (merrimanr) closes
apache/metron#1059
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/10271c41
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/10271c41
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/10271c41
Branch: refs/heads/master
Commit: 10271c41b1df2815041317a36f7b65def5dab831
Parents: 0e48427
Author: merrimanr
Authored: Fri Jun 15 08:17:37 2018 -0500
Committer: merrimanr
Committed: Fri Jun 15 08:17:37 2018 -0500
--
.../src/app/alerts/alert-details/alert-details.component.ts | 5 ++---
.../src/app/alerts/alerts-list/alerts-list.component.ts | 5 ++---
.../alerts/alerts-list/table-view/table-view.component.html | 2 +-
.../alerts/alerts-list/table-view/table-view.component.ts| 8 +++-
.../metron-alerts/src/app/service/update.service.ts | 4
metron-interface/metron-alerts/src/app/utils/constants.ts| 1 -
6 files changed, 8 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/10271c41/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index 2625502..ffd7dd0 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -29,7 +29,7 @@ import {Patch} from '../../model/patch';
import {AlertComment} from './alert-comment';
import {AuthenticationService} from '../../service/authentication.service';
import {MetronDialogBox} from '../../shared/metron-dialog-box';
-import {META_ALERTS_INDEX, META_ALERTS_SENSOR_TYPE} from
'../../utils/constants';
+import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
import {CommentAddRemoveRequest} from "../../model/comment-add-remove-request";
export enum AlertState {
@@ -127,7 +127,7 @@ export class AlertDetailsComponent implements OnInit {
this.alertId = params['guid'];
this.alertSourceType = params['source.type.field'];
this.alertIndex = params['index'];
- this.isMetaAlert = (this.alertIndex === META_ALERTS_INDEX &&
this.alertSourceType !== META_ALERTS_SENSOR_TYPE) ? true : false;
+ this.isMetaAlert = this.alertSourceType === META_ALERTS_SENSOR_TYPE;
this.getData();
});
};
@@ -195,7 +195,6 @@ export class AlertDetailsComponent implements OnInit {
let patchRequest = new PatchRequest();
patchRequest.guid = this.alertId;
patchRequest.sensorType = 'metaalert';
- patchRequest.index = META_ALERTS_INDEX;
patchRequest.patch = [new Patch('add', '/name', this.alertName)];
this.updateService.patch(patchRequest).subscribe(rep => {
http://git-wip-us.apache.org/repos/asf/metron/blob/10271c41/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
index a70f2b4..98459db 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alerts-list/alerts-list.component.ts
@@ -39,7 +39,7 @@ import {Filter} from '../../model/filter';
import {THREAT_SCORE_FIELD_NAME, TIMESTAMP_FIELD_NAME, ALL_TIME} from
'../../utils/constants';
import {TableViewComponent} from './table-view/table-view.component';
import {Pagination} from '../../model/pagination';
-import {META_ALERTS_SENSOR_TYPE, META_ALERTS_INDEX} from
'../../utils/constants';
+import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
import {MetaAlertService} from '../../service/meta-alert.service';
import {Facets} from '../../model/facets';
import { GlobalConfigService } from '../../service/global-config.service';
@@ -396,8 +396,7 @@ export class AlertsListComponent implements OnInit,
OnDestroy {
this.selectedAlerts = [];
this.selectedAlerts = [alert];
this.saveRefreshState();
-let sourceType = (alert.index === META_ALERTS_INDEX &&
!alert.source[this.globalConfig['source.type.field']])
-? META_ALERTS_SENSOR_TYPE :
alert.source[this.globalConfig['source.type.field']];
+let sourceType = alert.source[this.globalConfig['source.type.field']];
let url = '/alerts-list(dialog:details/' + sourceType + '/' +
alert.source.guid + '/' + alert.index + ')';
[44/53] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1416-upgrade-solr
Merge branch 'master' into feature/METRON-1416-upgrade-solr
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/4c4452e5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/4c4452e5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/4c4452e5
Branch: refs/heads/master
Commit: 4c4452e59c93306a049e6879f7b6fed2ee86c74e
Parents: 4768561
Author: cstella
Authored: Tue Jun 12 21:40:01 2018 -0400
Committer: cstella
Committed: Tue Jun 12 21:40:01 2018 -0400
--
.../dao/ElasticsearchSearchDao.java | 30 +--
.../ElasticsearchMetaAlertIntegrationTest.java | 223 ++-
2 files changed, 134 insertions(+), 119 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/4c4452e5/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
index 5725534..5cd0a4d 100644
---
a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
+++
b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchSearchDao.java
@@ -443,22 +443,24 @@ public class ElasticsearchSearchDao implements SearchDao {
private List getGroupResults(GroupRequest groupRequest, int
index, Aggregations aggregations, Map commonColumnMetadata) {
List groups = groupRequest.getGroups();
String field = groups.get(index).getField();
-Terms terms = aggregations.get(getGroupByAggregationName(field));
List searchResultGroups = new ArrayList<>();
-for(Bucket bucket: terms.getBuckets()) {
- GroupResult groupResult = new GroupResult();
- groupResult.setKey(formatKey(bucket.getKey(),
commonColumnMetadata.get(field)));
- groupResult.setTotal(bucket.getDocCount());
- Optional scoreField = groupRequest.getScoreField();
- if (scoreField.isPresent()) {
-Sum score =
bucket.getAggregations().get(getSumAggregationName(scoreField.get()));
-groupResult.setScore(score.getValue());
- }
- if (index < groups.size() - 1) {
-groupResult.setGroupedBy(groups.get(index + 1).getField());
-groupResult.setGroupResults(getGroupResults(groupRequest, index + 1,
bucket.getAggregations(), commonColumnMetadata));
+if(aggregations != null) {
+ Terms terms = aggregations.get(getGroupByAggregationName(field));
+ for (Bucket bucket : terms.getBuckets()) {
+GroupResult groupResult = new GroupResult();
+groupResult.setKey(formatKey(bucket.getKey(),
commonColumnMetadata.get(field)));
+groupResult.setTotal(bucket.getDocCount());
+Optional scoreField = groupRequest.getScoreField();
+if (scoreField.isPresent()) {
+ Sum score =
bucket.getAggregations().get(getSumAggregationName(scoreField.get()));
+ groupResult.setScore(score.getValue());
+}
+if (index < groups.size() - 1) {
+ groupResult.setGroupedBy(groups.get(index + 1).getField());
+ groupResult.setGroupResults(getGroupResults(groupRequest, index + 1,
bucket.getAggregations(), commonColumnMetadata));
+}
+searchResultGroups.add(groupResult);
}
- searchResultGroups.add(groupResult);
}
return searchResultGroups;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/4c4452e5/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
--
diff --git
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
index 99f0490..224ec56 100644
---
a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
+++
b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java
@@ -1,20 +1,21 @@
+
/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0
[19/53] [abbrv] metron git commit: Merge remote-tracking branch 'origin/master' into feature/METRON-1416-upgrade-solr
http://git-wip-us.apache.org/repos/asf/metron/blob/d0a4e4c0/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
--
diff --cc
metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
index 762a272,000..9f2414a
mode 100644,00..100644
---
a/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
+++
b/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrSearchDaoTest.java
@@@ -1,478 -1,0 +1,478 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.solr.dao;
+
+import org.apache.metron.common.Constants;
+import org.apache.metron.indexing.dao.AccessConfig;
+import org.apache.metron.indexing.dao.search.GetRequest;
+import org.apache.metron.indexing.dao.search.Group;
+import org.apache.metron.indexing.dao.search.GroupOrder;
+import org.apache.metron.indexing.dao.search.GroupRequest;
+import org.apache.metron.indexing.dao.search.GroupResponse;
+import org.apache.metron.indexing.dao.search.GroupResult;
+import org.apache.metron.indexing.dao.search.InvalidSearchException;
+import org.apache.metron.indexing.dao.search.SearchRequest;
+import org.apache.metron.indexing.dao.search.SearchResponse;
+import org.apache.metron.indexing.dao.search.SearchResult;
+import org.apache.metron.indexing.dao.search.SortField;
+import org.apache.metron.indexing.dao.update.Document;
+import org.apache.metron.solr.matcher.ModifiableSolrParamsMatcher;
+import org.apache.metron.solr.matcher.SolrQueryMatcher;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.FacetField;
+import org.apache.solr.client.solrj.response.FieldStatsInfo;
+import org.apache.solr.client.solrj.response.PivotField;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.IsCollectionContaining.hasItems;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({CollectionAdminRequest.class})
+public class SolrSearchDaoTest {
+
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ private SolrClient client;
+ private AccessConfig accessConfig;
+ private SolrSearchDao solrSearchDao;
+
+ @SuppressWarnings("unchecked")
+ @Before
+ public void setUp() throws Exception {
+client = mock(SolrClient.class);
+accessConfig = mock(AccessConfig.class);
+solrSearchDao = new SolrSearchDao(client, accessConfig);
+mockStatic(CollectionAdminRequest.class);
+
when(CollectionAdminRequest.listCollections(client)).thenReturn(Arrays.asList("bro",
"snort"));
+ }
+
+ @Test
+ public void searchShouldProperlyReturnSearchResponse() throws Exception
