[metron] branch feature/METRON-2088-support-hdp-3.1 updated: METRON-2169 Upgrade to Kafka 2.0.0 and Storm 1.2.1 (merrimanr) closes apache/metron#1490
This is an automated email from the ASF dual-hosted git repository. rmerriman pushed a commit to branch feature/METRON-2088-support-hdp-3.1 in repository https://gitbox.apache.org/repos/asf/metron.git The following commit(s) were added to refs/heads/feature/METRON-2088-support-hdp-3.1 by this push: new 15e5276 METRON-2169 Upgrade to Kafka 2.0.0 and Storm 1.2.1 (merrimanr) closes apache/metron#1490 15e5276 is described below commit 15e527601eee02d33f0a93388471e9525cbd927d Author: merrimanr AuthorDate: Tue Aug 27 13:48:09 2019 -0500 METRON-2169 Upgrade to Kafka 2.0.0 and Storm 1.2.1 (merrimanr) closes apache/metron#1490 --- dependencies_with_url.csv | 32 +++ metron-analytics/metron-maas-service/pom.xml | 4 + metron-analytics/metron-profiler-repl/pom.xml | 5 + metron-analytics/metron-profiler-spark/pom.xml | 6 ++ metron-analytics/metron-profiler-storm/pom.xml | 30 +++--- .../storm/integration/ProfilerIntegrationTest.java | 10 +- metron-contrib/metron-performance/pom.xml | 2 +- .../ansible/roles/metron-builder/tasks/build.yml | 2 +- .../ansible/roles/metron-builder/tasks/clean.yml | 2 +- metron-interface/metron-rest/pom.xml | 22 +++-- .../org/apache/metron/rest/config/KafkaConfig.java | 63 ++--- .../apache/metron/rest/config/ZookeeperConfig.java | 9 -- .../metron/rest/service/impl/KafkaServiceImpl.java | 59 ++-- .../org/apache/metron/rest/config/TestConfig.java | 33 --- .../rest/service/impl/KafkaServiceImplTest.java| 69 +- metron-platform/metron-common/pom.xml | 8 +- .../org/apache/metron/common/utils/KafkaUtils.java | 8 +- .../metron-elasticsearch-common/pom.xml| 21 + .../metron-enrichment-storm/pom.xml| 19 ++-- metron-platform/metron-enrichment/pom.xml | 6 ++ metron-platform/metron-hbase-server/pom.xml| 6 ++ .../metron-indexing/metron-indexing-common/pom.xml | 6 ++ .../metron-indexing/metron-indexing-storm/pom.xml | 5 + metron-platform/metron-integration-test/pom.xml| 15 ++- .../integration/components/KafkaComponent.java | 104 +++-- metron-platform/metron-management/pom.xml | 22 ++--- .../metron-parsing/metron-parsers-common/pom.xml | 21 ++--- .../metron-parsing/metron-parsers/pom.xml | 11 ++- .../metron/parsers/topology/ParserTopologyCLI.java | 2 +- metron-platform/metron-pcap-backend/pom.xml| 15 --- .../integration/PcapTopologyIntegrationTest.java | 9 +- metron-platform/metron-pcap/pom.xml| 29 +- .../metron-solr/metron-solr-common/pom.xml | 19 +--- .../metron-solr/metron-solr-storm/pom.xml | 48 +- .../metron-storm-kafka-override/pom.xml| 24 - metron-platform/metron-storm-kafka/pom.xml | 24 - metron-platform/metron-test-utilities/pom.xml | 2 +- .../metron-writer/metron-writer-common/pom.xml | 2 +- .../apache/metron/writer/kafka/KafkaWriter.java| 10 +- .../metron/writer/kafka/KafkaWriterTest.java | 4 +- metron-stellar/stellar-zeppelin/pom.xml| 6 ++ pom.xml| 26 ++ 42 files changed, 379 insertions(+), 441 deletions(-) diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index 0065f9b..9d7b0fa 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -5,6 +5,8 @@ org.objenesis:objenesis:jar:2.1:compile,Apache v2,http://objenesis.org/ org.ow2.asm:asm:jar:4.1:compile,BSD,http://asm.ow2.org/ org.ow2.asm:asm:jar:5.0.3:compile,BSD,http://asm.ow2.org/ org.ow2.asm:asm:jar:5.0.4:compile,BSD,http://asm.ow2.org/ +asm:asm-commons:jar:3.1:compile,BSD,https://asm.ow2.io/ +asm:asm-tree:jar:3.1:compile,BSD,https://asm.ow2.io/ org.reflections:reflections:jar:0.9.10:compile,BSD,https://github.com/ronmamo/reflections org.javassist:javassist:jar:3.19.0-GA:compile,Apache v2,https://github.com/jboss-javassist/javassist org.javassist:javassist:jar:3.17.1-GA:compile,Apache v2,https://github.com/jboss-javassist/javassist @@ -43,6 +45,7 @@ javax.annotation:jsr250-api:jar:1.0:compile,COMMON DEVELOPMENT AND DISTRIBUTION javax.annotation:javax.annotation-api:jar:1.3.2:compile,CDDL 1.1,https://github.com/javaee/javax.annotation/ javax.annotation:javax.annotation-api:jar:1.2:compile,CDDL 1.1,https://github.com/javaee/javax.annotation/ javax.mail:mail:jar:1.4:compile,Common Development and Distribution License (CDDL) v1.0,https://glassfish.dev.java.net/javaee5/mail/ +javax.mail:mail:jar:1.4.1:compile,Common Development and Distribution License (CDDL) v1.0,https://glassfish.dev.java.net/javaee5/mail/ javax.servlet:javax.servlet-api:jar:3.1.0:compile,CDDL,http://servlet-spec.java.net javax.ws.rs:javax.ws.rs-api:jar:2.0.1:compile,CDDL 1.1,https://github.com/jax-rs/api javax.xml.bind:jaxb-api:jar
[metron] branch master updated: METRON-2149 Shaded jar classifier is not consistent (merrimanr) closes apache/metron#1436
This is an automated email from the ASF dual-hosted git repository. rmerriman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron.git The following commit(s) were added to refs/heads/master by this push: new e6ca3c7 METRON-2149 Shaded jar classifier is not consistent (merrimanr) closes apache/metron#1436 e6ca3c7 is described below commit e6ca3c71d8120516afcf816be2cda81093088b72 Author: merrimanr AuthorDate: Fri Aug 23 16:42:01 2019 -0500 METRON-2149 Shaded jar classifier is not consistent (merrimanr) closes apache/metron#1436 --- metron-analytics/metron-maas-common/pom.xml| 1 + metron-analytics/metron-maas-service/pom.xml | 6 ++ metron-analytics/metron-profiler-client/pom.xml| 8 ++ .../src/main/assembly/assembly.xml | 2 +- metron-analytics/metron-profiler-common/pom.xml| 6 ++ metron-analytics/metron-profiler-repl/pom.xml | 8 ++ .../src/main/assembly/assembly.xml | 2 +- metron-analytics/metron-profiler-spark/pom.xml | 3 + .../src/main/assembly/assembly.xml | 2 +- .../src/main/scripts/start_batch_profiler.sh | 4 +- metron-analytics/metron-profiler-storm/pom.xml | 8 ++ .../src/main/scripts/start_profiler_topology.sh| 3 +- metron-analytics/metron-statistics/pom.xml | 3 + metron-contrib/metron-performance/pom.xml | 8 ++ .../src/main/assembly/assembly.xml | 2 +- .../src/main/scripts/load_tool.sh | 2 +- .../common-services/METRON/CURRENT/metainfo.xml| 3 + .../packaging/docker/rpm-docker/SPECS/metron.spec | 35 +++-- .../packaging/docker/rpm-docker/pom.xml| 6 ++ metron-interface/metron-rest/pom.xml | 22 ++ .../metron-rest/src/main/assembly/assembly.xml | 2 +- .../metron-rest/src/main/scripts/metron-rest.sh| 2 + metron-platform/elasticsearch-shaded/pom.xml | 2 + .../metron-common-storm/pom.xml| 6 ++ metron-platform/metron-common/pom.xml | 51 +++-- .../metron-common/src/main/assembly/assembly.xml | 2 +- .../metron-common/src/main/scripts/stellar | 5 +- .../src/main/scripts/zk_load_configs.sh| 3 +- metron-platform/metron-data-management/pom.xml | 89 ++ .../src/main/assembly/assembly.xml | 2 +- .../src/main/scripts/flatfile_loader.sh| 7 +- .../src/main/scripts/flatfile_summarizer.sh| 7 +- .../src/main/scripts/maxmind_enrichment_load.sh| 10 +-- .../src/main/scripts/prune_hdfs_files.sh | 2 +- .../src/main/scripts/threatintel_bulk_prune.sh | 2 +- .../src/main/scripts/threatintel_taxii_load.sh | 2 +- .../metron-elasticsearch-common/pom.xml| 27 ++- .../metron-elasticsearch-storm/pom.xml | 19 + .../main/scripts/start_elasticsearch_topology.sh | 5 +- .../metron-enrichment-common/pom.xml | 1 + .../metron-enrichment-storm/pom.xml| 6 ++ .../src/main/scripts/start_enrichment_topology.sh | 3 +- metron-platform/metron-hbase-client/pom.xml| 2 + metron-platform/metron-hbase-server/pom.xml| 6 ++ .../metron-indexing/metron-indexing-common/pom.xml | 11 +-- .../metron-indexing/metron-indexing-storm/pom.xml | 6 ++ .../src/main/scripts/start_hdfs_topology.sh| 3 +- metron-platform/metron-integration-test/pom.xml| 1 + metron-platform/metron-management/pom.xml | 3 + .../src/main/assembly/assembly.xml | 2 +- .../metron-parsing/metron-parsers-common/pom.xml | 6 ++ .../metron-parsing/metron-parsers/pom.xml | 6 ++ .../metron-parsing/metron-parsing-storm/pom.xml| 6 ++ .../src/main/scripts/start_parser_topology.sh | 8 +- metron-platform/metron-pcap-backend/pom.xml| 40 ++ .../src/main/assembly/assembly.xml | 2 +- .../src/main/scripts/pcap_inspector.sh | 2 +- .../src/main/scripts/pcap_query.sh | 2 +- .../src/main/scripts/start_pcap_topology.sh| 6 +- metron-platform/metron-pcap/pom.xml| 14 .../metron-solr/metron-solr-common/pom.xml | 6 ++ .../metron-solr/metron-solr-storm/pom.xml | 6 ++ .../src/main/scripts/start_solr_topology.sh| 3 +- .../metron-storm-kafka-override/pom.xml| 2 + metron-platform/metron-storm-kafka/pom.xml | 1 + .../metron-writer/metron-writer-common/pom.xml | 45 ++- .../metron-writer/metron-writer-storm/pom.xml | 6 ++ metron-stellar/stellar-common/pom.xml | 10 ++- .../stellar-common/src/main/assembly/assembly.xml | 12 --- .../src/main/scripts/deployed/stellar | 4 +- metron-stellar/stellar-zeppelin/pom.xml| 1 + 71 files changed, 444 insertions(+), 167 deletions(-) diff --git a/metron-analytics
[metron] branch master updated: METRON-2148 Stellar REST POST function (merrimanr) closes apache/metron#1440
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 42ce5ad METRON-2148 Stellar REST POST function (merrimanr) closes
apache/metron#1440
42ce5ad is described below
commit 42ce5add9ae5b20bb3667e81e8aaa148f48a4d42
Author: merrimanr
AuthorDate: Thu Jul 11 08:16:30 2019 -0500
METRON-2148 Stellar REST POST function (merrimanr) closes apache/metron#1440
---
metron-stellar/stellar-common/README.md| 98 ++-
.../metron/stellar/common/utils/JSONUtils.java | 6 +-
.../metron/stellar/dsl/functions/RestConfig.java | 23 +-
.../stellar/dsl/functions/RestFunctions.java | 657 +
.../functions/RestFunctionsIntegrationTest.java| 490 +++
.../stellar/dsl/functions/RestFunctionsTest.java | 449 +++---
6 files changed, 1095 insertions(+), 628 deletions(-)
diff --git a/metron-stellar/stellar-common/README.md
b/metron-stellar/stellar-common/README.md
index 5e48b1c..9f1634b 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -963,6 +963,16 @@ Where:
* Input:
* url - URI to the REST service
* rest_config - Optional - Map (in curly braces) of name:value pairs, each
overriding the global config parameter of the same name. Default is the empty
Map, meaning no overrides.
+* query_parameters - Optional - Map (in curly braces) of name:value pairs
that will be added to the request as query parameters
+ * Returns: JSON results as a Map
+
+### `REST_POST`
+ * Description: Performs a REST POST request and parses the JSON results into
a map.
+ * Input:
+* url - URI to the REST service
+* post_data - POST data that will be sent in the POST request. Must be
well-formed JSON unless the 'enforce.json' property is set to false.
+* rest_config - Optional - Map (in curly braces) of name:value pairs, each
overriding the global config parameter of the same name. Default is the empty
Map, meaning no overrides.
+* query_parameters - Optional - Map (in curly braces) of name:value pairs
that will be added to the request as query parameters
* Returns: JSON results as a Map
### `ROUND`
@@ -1649,15 +1659,64 @@ that specify what should be included when searching for
Stellar functions.
## Stellar REST Client
-Stellar provides a REST Client with the `REST_GET` function. This function
depends on the Apache HttComponents library for
-executing Http requests. The syntax is:
+Stellar provides a REST Client with the `REST_GET` and `REST_POST` functions.
This function depends on the Apache HttComponents library for
+executing Http requests.
+
+### REST GET Syntax
+The REST_GET function requires a URI along with an optional configuration and
an optional map of query parameters. The syntax is:
```
-REST_GET( uri , optional config )
+REST_GET( uri , optional config , optional query parameters )
+```
+
+### REST POST Syntax
+The REST_POST function requires a URI and POST data along with an optional
configuration and an optional map of query parameters. The syntax is:
+```
+REST_POST( uri , data, optional config , optional query parameters )
```
### Configuration
-The second argument is an optional Map of settings. The following settings
are available:
+Stellar REST functions can be configured several different ways. Sensible
defaults are set for applicable settings with the option to override settings
at different levels.
+For REST_GET, configuration settings are applied in this order (last has
highest priority):
+1. Default settings
+2. Settings stored in the Global Config for all Stellar REST functions
+3. Settings stored in the Global Config for all Stellar REST_GET calls
+4. Settings passed into the function call as an argument
+
+For REST_POST, configuration settings are applied in this order (last has
highest priority):
+1. Default settings
+2. Settings stored in the Global Config for all Stellar REST functions
+3. Settings stored in the Global Config for all Stellar REST_POST calls
+4. Settings passed into the function call as an argument
+
+For example, assume the Global Config is set to:
+```
+{
+ "stellar.rest.settings": {
+"proxy.basic.auth.user": "global_proxy_user",
+"basic.auth.user": "global_user",
+"empty.content.override": "global content override"
+ },
+ "stellar.rest.get.settings": {
+"basic.auth.user": "rest_get_user",
+"empty.content.override": "rest get content override"
+ }
+}
+```
+and the function call is:
+```
+REST_GET('some uri', { "empty.content.override": "function config override" } )
+```
+After the various settings are applied in order of priority, the final
[metron] branch master updated: METRON-2061 Solr documents with date fields cannot be updated with Dao classes (merrimanr) closes apache/metron#1374
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 8792882 METRON-2061 Solr documents with date fields cannot be updated
with Dao classes (merrimanr) closes apache/metron#1374
8792882 is described below
commit 87928824509c5f6ab314375be720dd64d8361cf4
Author: merrimanr
AuthorDate: Tue Jul 2 08:47:27 2019 -0500
METRON-2061 Solr documents with date fields cannot be updated with Dao
classes (merrimanr) closes apache/metron#1374
---
.../metron/indexing/dao/update/PatchException.java | 27 +++
.../metron/indexing/dao/update/PatchOperation.java | 28 +++
.../metron/indexing/dao/update/PatchUtils.java | 105
.../metron/indexing/dao/update/UpdateDao.java | 3 +-
.../metron/indexing/dao/update/PatchUtilsTest.java | 263 +
5 files changed, 424 insertions(+), 2 deletions(-)
diff --git
a/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchException.java
b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchException.java
new file mode 100644
index 000..c621cfa
--- /dev/null
+++
b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchException.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.metron.indexing.dao.update;
+
+public class PatchException extends RuntimeException {
+
+ public PatchException(String message) {
+super(message);
+ }
+}
diff --git
a/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchOperation.java
b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchOperation.java
new file mode 100644
index 000..9b0e92d
--- /dev/null
+++
b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchOperation.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.indexing.dao.update;
+
+enum PatchOperation {
+ ADD,
+ REPLACE,
+ REMOVE,
+ COPY,
+ MOVE,
+ TEST;
+}
diff --git
a/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchUtils.java
b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchUtils.java
new file mode 100644
index 000..e5c5117
--- /dev/null
+++
b/metron-platform/metron-indexing/metron-indexing-common/src/main/java/org/apache/metron/indexing/dao/update/PatchUtils.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required b
[metron] branch master updated: METRON-2156 Remove Storm dependency from metron-hbase (merrimanr) closes apache/metron#1441
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new a2a46e6 METRON-2156 Remove Storm dependency from metron-hbase
(merrimanr) closes apache/metron#1441
a2a46e6 is described below
commit a2a46e6567c78e5066c0d3360d7838d1a5851fa7
Author: merrimanr
AuthorDate: Thu Jun 20 08:29:21 2019 -0500
METRON-2156 Remove Storm dependency from metron-hbase (merrimanr) closes
apache/metron#1441
---
metron-analytics/metron-profiler-client/pom.xml| 4 +-
.../metron/profiler/client/ProfileWriter.java | 2 +-
metron-analytics/metron-profiler-common/pom.xml| 2 +-
.../metron/profiler/hbase/ColumnBuilder.java | 2 +-
.../profiler/hbase/ValueOnlyColumnBuilder.java | 2 +-
metron-analytics/metron-profiler-spark/pom.xml | 4 +-
metron-analytics/metron-profiler-storm/pom.xml | 13 +-
.../org/apache/metron/hbase/bolt/BatchHelper.java | 0
.../org/apache/metron/hbase/bolt/HBaseBolt.java| 2 +-
.../metron/hbase/bolt/mapper/HBaseMapper.java | 1 +
.../metron/profiler/storm/ProfileHBaseMapper.java | 2 +-
.../apache/metron/hbase/bolt/HBaseBoltTest.java| 4 +-
.../apache/metron/hbase/bolt/mapper}/Widget.java | 2 +-
.../metron/hbase/bolt/mapper}/WidgetMapper.java| 5 +-
metron-interface/metron-rest/pom.xml | 7 +-
metron-platform/metron-data-management/pom.xml | 4 +-
.../metron-elasticsearch-common/pom.xml| 2 +-
.../integration/mock/MockHBaseConnector.java | 52
metron-platform/metron-enrichment/pom.xml | 4 +-
.../metron-hbase/{ => metron-hbase-common}/pom.xml | 69 +
.../java/org/apache/metron/hbase}/ColumnList.java | 2 +-
.../metron/hbase}/HBaseProjectionCriteria.java | 2 +-
.../org/apache/metron/hbase/HTableProvider.java| 0
.../java/org/apache/metron/hbase}/IColumn.java | 2 +-
.../java/org/apache/metron/hbase}/ICounter.java| 2 +-
.../java/org/apache/metron/hbase/TableConfig.java | 0
.../org/apache/metron/hbase/TableProvider.java | 0
.../apache/metron/hbase/client/HBaseClient.java| 4 +-
.../metron/hbase/client/HBaseClientTest.java | 126 +++--
.../metron/hbase/mock/MockHBaseTableProvider.java | 0
.../org/apache/metron/hbase/mock/MockHTable.java | 0
.../src/test/resources/log4j.properties| 0
metron-platform/metron-hbase/pom.xml | 283 +++--
.../java/org/apache/metron/hbase/Connector.java| 36 ---
.../org/apache/metron/hbase/HTableConnector.java | 157
.../org/apache/metron/hbase/TupleTableConfig.java | 275
.../metron-indexing/metron-indexing-common/pom.xml | 7 +-
.../metron-parsing/metron-parsers-common/pom.xml | 7 -
.../metron-parsing/metron-parsing-storm/pom.xml| 2 +-
metron-platform/metron-pcap/pom.xml| 2 +-
.../metron-solr/metron-solr-common/pom.xml | 2 +-
.../metron-writer/metron-writer-common/pom.xml | 2 +-
42 files changed, 119 insertions(+), 975 deletions(-)
diff --git a/metron-analytics/metron-profiler-client/pom.xml
b/metron-analytics/metron-profiler-client/pom.xml
index a0cfa1a..04d9740 100644
--- a/metron-analytics/metron-profiler-client/pom.xml
+++ b/metron-analytics/metron-profiler-client/pom.xml
@@ -65,7 +65,7 @@
org.apache.metron
-metron-hbase
+metron-hbase-common
${project.parent.version}
@@ -84,7 +84,7 @@
org.apache.metron
-metron-hbase
+metron-hbase-common
${project.parent.version}
test
test-jar
diff --git
a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/ProfileWriter.java
b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/ProfileWriter.java
index 38f1c3e..4e00164 100644
---
a/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/ProfileWriter.java
+++
b/metron-analytics/metron-profiler-client/src/test/java/org/apache/metron/profiler/client/ProfileWriter.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.metron.hbase.HTableProvider;
-import org.apache.metron.hbase.bolt.mapper.ColumnList;
+import org.apache.metron.hbase.ColumnList;
import org.apache.metron.hbase.client.HBaseClient;
import org.apache.metron.profiler.ProfileMeasurement;
import org.apache.metron.profiler.ProfilePeriod;
diff --git a/metron-analytics/metron-profiler-common/pom.xml
b/metron-analytics/metron-profiler-common/pom.xml
in
[metron] branch master updated: METRON-2073 Create in-memory use case for enrichment with map type and flatfile summarizer (merrimanr) closes apache/metron#1399
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 38b8a78 METRON-2073 Create in-memory use case for enrichment with map
type and flatfile summarizer (merrimanr) closes apache/metron#1399
38b8a78 is described below
commit 38b8a7824a3ceed829215c8ce4ecad963ebf449e
Author: merrimanr
AuthorDate: Mon Jun 17 13:32:44 2019 -0500
METRON-2073 Create in-memory use case for enrichment with map type and
flatfile summarizer (merrimanr) closes apache/metron#1399
---
.../metron/enrichment/cache/ObjectCache.java | 123 +
.../metron/enrichment/cache/ObjectCacheConfig.java | 115
.../enrichment/stellar/EnrichmentObjectGet.java| 101 ++
.../metron/enrichment/stellar/ObjectGet.java | 94 ++---
.../ObjectCacheTest.java} | 63 ++---
.../EnrichmentObjectGetIntegrationTest.java| 72 ++
.../stellar/EnrichmentObjectGetTest.java | 152 +
.../stellar/ObjectGetIntegrationTest.java | 70 ++
.../metron/enrichment/stellar/ObjectGetTest.java | 115 +---
9 files changed, 757 insertions(+), 148 deletions(-)
diff --git
a/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/ObjectCache.java
b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/ObjectCache.java
new file mode 100644
index 000..9d22bfc
--- /dev/null
+++
b/metron-platform/metron-enrichment/metron-enrichment-common/src/main/java/org/apache/metron/enrichment/cache/ObjectCache.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.enrichment.cache;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.CacheLoader;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.common.utils.SerDeUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+public class ObjectCache {
+ private static final Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ protected LoadingCache cache;
+ private static ReadWriteLock lock = new ReentrantReadWriteLock();
+
+ public class Loader implements CacheLoader {
+FileSystem fs;
+ObjectCacheConfig objectCacheConfig;
+
+public Loader(Configuration hadoopConfig, ObjectCacheConfig
objectCacheConfig) throws IOException {
+ this.fs = FileSystem.get(hadoopConfig);
+ this.objectCacheConfig = objectCacheConfig;
+}
+
+@Override
+public Object load(String s) throws Exception {
+ LOG.debug("Loading object from path '{}'", s);
+ if (StringUtils.isEmpty(s)) {
+throw new IllegalArgumentException("Path cannot be empty");
+ }
+ Object object = null;
+ Path p = new Path(s);
+ if (fs.exists(p)) {
+if (fs.getFileStatus(p).getLen() <=
objectCacheConfig.getMaxFileSize()) {
+ try (InputStream is = new BufferedInputStream(fs.open(p))) {
+byte[] serialized = IOUtils.toByteArray(is);
+if (serialized.length > 0) {
+ object = SerDeUtils.fromBytes(serialized, Object.class);
+}
+ }
+} else {
+ throw new IllegalArgumentException(String.format("File at path '%s'
is larger than the configured max file size of %s", p,
objectCacheConfig.getMaxFileSize()));
+}
+ } else {
+throw new
[metron] branch feature/METRON-1856-parser-aggregation updated: METRON-2117 [UI] Aligning models to grouping feature (tiborm via merrimanr) closes apache/metron#1412
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch feature/METRON-1856-parser-aggregation
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to
refs/heads/feature/METRON-1856-parser-aggregation by this push:
new 1ad26a1 METRON-2117 [UI] Aligning models to grouping feature (tiborm
via merrimanr) closes apache/metron#1412
1ad26a1 is described below
commit 1ad26a103b62e66d85473f1700142f1bc3b9b1a6
Author: tiborm
AuthorDate: Wed Jun 12 10:31:18 2019 -0500
METRON-2117 [UI] Aligning models to grouping feature (tiborm via merrimanr)
closes apache/metron#1412
---
metron-interface/metron-config/src/app/index.ts| 2 -
.../src/app/model/parse-message-request.ts | 4 +-
.../src/app/model/sensor-parser-config-history.ts | 29 ++--
.../src/app/model/sensor-parser-context.ts | 4 +-
.../src/app/model/sensor-parser-info.ts| 4 +-
.../src/app/model/topology-response.ts | 4 ++
.../metron-config/src/app/model/topology-status.ts | 12
.../models/parser-config.model.ts} | 83 +-
.../src/app/sensors/models/parser-group.model.ts | 66 +
.../models/parser-meta-info.model.ts} | 26 ---
.../models/parser.model.ts}| 11 ++-
.../sensor-field-schema.component.spec.ts | 6 +-
.../sensor-field-schema.component.ts | 4 +-
.../sensor-grok/sensor-grok.component.spec.ts | 4 +-
.../sensors/sensor-grok/sensor-grok.component.ts | 4 +-
...sensor-parser-config-readonly.component.spec.ts | 10 +--
.../sensor-parser-config-readonly.component.ts | 6 +-
.../sensor-raw-json.component.spec.ts | 8 +--
.../sensor-raw-json/sensor-raw-json.component.ts | 4 +-
.../sensor-storm-settings.component.spec.ts| 10 +--
.../sensor-storm-settings.component.ts | 8 +--
.../sensor-parser-config-history.service.ts| 10 +--
.../service/sensor-parser-config.service.spec.ts | 21 +++---
.../src/app/service/stellar.service.spec.ts| 4 +-
.../src/app/shared/metron-alerts.spec.ts | 8 +--
.../metron-config/src/app/shared/metron-alerts.ts | 7 +-
26 files changed, 260 insertions(+), 99 deletions(-)
diff --git a/metron-interface/metron-config/src/app/index.ts
b/metron-interface/metron-config/src/app/index.ts
index 1639c33..16c868f 100644
--- a/metron-interface/metron-config/src/app/index.ts
+++ b/metron-interface/metron-config/src/app/index.ts
@@ -18,5 +18,3 @@
export * from './environment';
export * from './app.component';
export * from './app.routes';
-export * from './app.config';
-export * from './app.config.interface';
diff --git
a/metron-interface/metron-config/src/app/model/parse-message-request.ts
b/metron-interface/metron-config/src/app/model/parse-message-request.ts
index 053d7ad..b58db63 100644
--- a/metron-interface/metron-config/src/app/model/parse-message-request.ts
+++ b/metron-interface/metron-config/src/app/model/parse-message-request.ts
@@ -15,9 +15,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import {SensorParserConfig} from './sensor-parser-config';
+import {ParserConfigModel} from '../sensors/models/parser-config.model';
export class ParseMessageRequest {
- sensorParserConfig: SensorParserConfig;
+ sensorParserConfig: ParserConfigModel;
grokStatement: string;
sampleData: string;
}
diff --git
a/metron-interface/metron-config/src/app/model/sensor-parser-config-history.ts
b/metron-interface/metron-config/src/app/model/sensor-parser-config-history.ts
index 4854001..a4eda68 100644
---
a/metron-interface/metron-config/src/app/model/sensor-parser-config-history.ts
+++
b/metron-interface/metron-config/src/app/model/sensor-parser-config-history.ts
@@ -15,16 +15,37 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import {SensorParserConfig} from './sensor-parser-config';
+import {ParserConfigModel} from '../sensors/models/parser-config.model';
export class SensorParserConfigHistory {
sensorName: string;
createdBy: string;
modifiedBy: string;
createdDate: string;
modifiedByDate: string;
- config: SensorParserConfig;
+ config: ParserConfigModel;
+ status: string;
+ latency: string;
+ throughput: string;
- constructor() {
-this.config = new SensorParserConfig();
+ constructor() {}
+
+ setConfig(config) {
+this.config = new ParserConfigModel(config);
+ }
+
+ clone(): SensorParserConfigHistory {
+const clone = new SensorParserConfigHistory();
+
+clone.sensorName = this.sensorName;
+clone.createdBy = this.createdBy;
+clone.modifiedBy = this.modifiedBy;
+clone.createdDate = this.createdDate;
+clone.modifiedByDate = this.modifiedByDate;
+clone.config = this.config.clone
[metron] branch feature/METRON-1856-parser-aggregation updated: METRON-2116 [UI] Removing redundant AppConfigService (tiborm via merrimanr) closes apache/metron#1411
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch feature/METRON-1856-parser-aggregation
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to
refs/heads/feature/METRON-1856-parser-aggregation by this push:
new 6a19784 METRON-2116 [UI] Removing redundant AppConfigService (tiborm
via merrimanr) closes apache/metron#1411
6a19784 is described below
commit 6a197849bee427e33a373d2add5f304e7ad0c977
Author: tiborm
AuthorDate: Wed Jun 12 08:29:57 2019 -0500
METRON-2116 [UI] Removing redundant AppConfigService (tiborm via merrimanr)
closes apache/metron#1411
---
.../metron-config/src/app/app.config.interface.ts | 21 -
.../metron-config/src/app/app.config.ts| 26 --
.../src/app/login/login.component.spec.ts | 5 +++--
.../src/app/service/global-config.service.spec.ts | 5 ++---
.../src/app/service/hdfs.service.spec.ts | 1 -
.../sensor-enrichment-config.service.spec.ts | 1 -
6 files changed, 5 insertions(+), 54 deletions(-)
diff --git a/metron-interface/metron-config/src/app/app.config.interface.ts
b/metron-interface/metron-config/src/app/app.config.interface.ts
deleted file mode 100644
index 4845b46..000
--- a/metron-interface/metron-config/src/app/app.config.interface.ts
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-export interface IAppConfig {
- apiEndpoint: string;
-}
diff --git a/metron-interface/metron-config/src/app/app.config.ts
b/metron-interface/metron-config/src/app/app.config.ts
deleted file mode 100644
index abd378c..000
--- a/metron-interface/metron-config/src/app/app.config.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { InjectionToken } from '@angular/core';
-import {IAppConfig} from './app.config.interface';
-
-export let APP_CONFIG = new InjectionToken('app.config');
-
-export const METRON_REST_CONFIG: IAppConfig = {
-apiEndpoint: '/api/v1'
-};
diff --git
a/metron-interface/metron-config/src/app/login/login.component.spec.ts
b/metron-interface/metron-config/src/app/login/login.component.spec.ts
index eac383c..f48d897 100644
--- a/metron-interface/metron-config/src/app/login/login.component.spec.ts
+++ b/metron-interface/metron-config/src/app/login/login.component.spec.ts
@@ -21,7 +21,8 @@ import { LoginComponent } from './login.component';
import { Observable } from 'rxjs';
import { ActivatedRoute, Params } from '@angular/router';
import { LoginModule } from './login.module';
-import { APP_CONFIG, METRON_REST_CONFIG } from '../app.config';
+import { AppConfigService } from 'app/service/app-config.service';
+import { MockAppConfigService } from 'app/service/mock.app-config.service';
class MockAuthenticationService {
public login(username: string, password: string, onError): void {
@@ -59,7 +60,7 @@ describe('LoginComponent', () => {
providers: [
{ provide: ActivatedRoute, useClass: MockActivatedRoute },
{ provide: AuthenticationService, useClass: MockAuthenticationService
},
-{ provide: APP_CONFIG, useValue: METRON_REST_CONFIG }
+{ provide: AppConfigService, useValue: MockAppConfigService }
]
});
fixture = TestBed.createComponent(LoginComponent);
diff --git
a/metron-int
[metron] branch feature/METRON-1856-parser-aggregation updated: METRON-2114 [UI] Moving components to sensor parser module (tiborm via merrimanr) closes apache/metron#1410
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch feature/METRON-1856-parser-aggregation
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to
refs/heads/feature/METRON-1856-parser-aggregation by this push:
new bfc496c METRON-2114 [UI] Moving components to sensor parser module
(tiborm via merrimanr) closes apache/metron#1410
bfc496c is described below
commit bfc496cc1267a3c5c6939fb6704fd30582c4057c
Author: tiborm
AuthorDate: Tue Jun 11 07:57:50 2019 -0500
METRON-2114 [UI] Moving components to sensor parser module (tiborm via
merrimanr) closes apache/metron#1410
---
.../metron-config/src/app/app.module.ts| 93 --
.../src/app/sensors/sensors.module.ts | 60 ++
2 files changed, 111 insertions(+), 42 deletions(-)
diff --git a/metron-interface/metron-config/src/app/app.module.ts
b/metron-interface/metron-config/src/app/app.module.ts
index 89aa937..5a730ed 100644
--- a/metron-interface/metron-config/src/app/app.module.ts
+++ b/metron-interface/metron-config/src/app/app.module.ts
@@ -15,55 +15,64 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import {APP_INITIALIZER, NgModule} from '@angular/core';
-import {FormsModule, ReactiveFormsModule} from '@angular/forms';
-import {BrowserModule} from '@angular/platform-browser';
-import {HttpClientModule, HTTP_INTERCEPTORS} from '@angular/common/http';
-import { Router } from '@angular/router';
-import {AppComponent} from './app.component';
-import {SensorParserConfigService} from
'./service/sensor-parser-config.service';
-import {KafkaService} from './service/kafka.service';
-import {GrokValidationService} from './service/grok-validation.service';
-import {StellarService} from './service/stellar.service';
-import {MetronAlerts} from './shared/metron-alerts';
-import {NavbarComponent} from './navbar/navbar.component';
-import {VerticalNavbarComponent} from
'./verticalnavbar/verticalnavbar.component';
-import {MetronConfigRoutingModule} from './app.routes';
-import {AuthenticationService} from './service/authentication.service';
-import {AuthGuard} from './shared/auth-guard';
-import {LoginGuard} from './shared/login-guard';
-import {SensorParserConfigModule} from
'./sensors/sensor-parser-config/sensor-parser-config.module';
-import {SensorParserConfigReadonlyModule} from
'./sensors/sensor-parser-config-readonly/sensor-parser-config-readonly.module';
-import {SensorParserListModule} from
'./sensors/sensor-parser-list/sensor-parser-list.module';
-import {MetronDialogBox} from './shared/metron-dialog-box';
-import {GeneralSettingsModule} from
'./general-settings/general-settings.module';
-import {SensorEnrichmentConfigService} from
'./service/sensor-enrichment-config.service';
-import {GlobalConfigService} from './service/global-config.service';
-import {APP_CONFIG, METRON_REST_CONFIG} from './app.config';
-import {StormService} from './service/storm.service';
-import {SensorParserConfigHistoryService} from
'./service/sensor-parser-config-history.service';
-import {SensorIndexingConfigService} from
'./service/sensor-indexing-config.service';
-import {HdfsService} from './service/hdfs.service';
+import { StoreModule, MetaReducer } from '@ngrx/store';
+import { StoreDevtoolsModule } from '@ngrx/store-devtools';
+import { EffectsModule } from '@ngrx/effects'
+import { SensorsModule } from './sensors/sensors.module';
+import { storeFreeze } from 'ngrx-store-freeze';
+import { environment } from '../environments/environment';
+
+import { APP_INITIALIZER, NgModule } from '@angular/core';
+import { FormsModule, ReactiveFormsModule } from '@angular/forms';
+import { BrowserModule } from '@angular/platform-browser';
+import { HttpClientModule, HTTP_INTERCEPTORS } from '@angular/common/http';
+import { AppComponent } from './app.component';
+import { MetronAlerts } from './shared/metron-alerts';
+import { NavbarComponent } from './navbar/navbar.component';
+import { VerticalNavbarComponent } from
'./verticalnavbar/verticalnavbar.component';
+import { MetronConfigRoutingModule } from './app.routes';
+import { AuthenticationService } from './service/authentication.service';
+import { AuthGuard } from './shared/auth-guard';
+import { LoginGuard } from './shared/login-guard';
+import { MetronDialogBox } from './shared/metron-dialog-box';
+import { GeneralSettingsModule } from
'./general-settings/general-settings.module';
+import { GlobalConfigService } from './service/global-config.service';
import { DefaultHeadersInterceptor } from
'./http-interceptors/default-headers.interceptor';
-import {AppConfigService} from './service/app-config.service';
+import {AppConfigService } from './service/app-config.service';
+
+export const metaReducers: MetaReducer<{}>[] = !environment.production
+? [storeFreeze]
+: [];
export fu
[metron] branch master updated: METRON-2153 ParserIntegrationTest should print failed messages (merrimanr) closes apache/metron#1438
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new a1487ec METRON-2153 ParserIntegrationTest should print failed
messages (merrimanr) closes apache/metron#1438
a1487ec is described below
commit a1487ec104c0ebb7e3383499b445726dbe5de142
Author: merrimanr
AuthorDate: Wed Jun 5 17:36:05 2019 -0500
METRON-2153 ParserIntegrationTest should print failed messages (merrimanr)
closes apache/metron#1438
---
.../java/org/apache/metron/common/error/MetronError.java| 4
.../java/org/apache/metron/parsers/bolt/ParserBolt.java | 6 +-
.../parsers/integration/validation/StormParserDriver.java | 13 +
3 files changed, 22 insertions(+), 1 deletion(-)
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java
index 46fe3b2..fe6a161 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/error/MetronError.java
@@ -106,6 +106,10 @@ public class MetronError {
return throwable != null ? Optional.of(throwable) : Optional.empty();
}
+ public List getRawMessages() {
+return rawMessages;
+ }
+
/**
* Serializes the MetronError into a JSON object.
*
diff --git
a/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index 2837fcb..f6715a5 100644
---
a/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++
b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -255,7 +255,7 @@ public class ParserBolt extends ConfiguredParserBolt
implements Serializable {
, sensorParserConfig.getRawMessageStrategyConfig()
);
ParserRunnerResults parserRunnerResults =
parserRunner.execute(sensorType, rawMessage, parserConfigurations);
- parserRunnerResults.getErrors().forEach(error ->
StormErrorUtils.handleError(collector, error));
+ parserRunnerResults.getErrors().forEach(error -> handleError(collector,
error));
WriterHandler writer = sensorToWriterMap.get(sensorType);
int numWritten = 0;
@@ -326,6 +326,10 @@ public class ParserBolt extends ConfiguredParserBolt
implements Serializable {
.withThrowable(ex)
.withSensorType(Collections.singleton(sensorType))
.addRawMessage(originalMessage);
+handleError(collector, error);
+ }
+
+ protected void handleError(OutputCollector collector, MetronError error) {
StormErrorUtils.handleError(collector, error);
}
diff --git
a/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
index 0d46d73..764a445 100644
---
a/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
+++
b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
@@ -35,6 +35,7 @@ import org.apache.commons.lang.SerializationUtils;
import org.apache.metron.common.configuration.IndexingConfigurations;
import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.error.MetronError;
import org.apache.metron.common.writer.BulkMessageWriter;
import org.apache.metron.common.writer.BulkMessage;
import org.apache.metron.common.writer.BulkWriterResponse;
@@ -112,6 +113,18 @@ public class StormParserDriver extends ParserDriver {
LOG.error("Error parsing message: " + ex.getMessage(), ex);
}
+@Override
+protected void handleError(OutputCollector collector, MetronError error) {
+ for(Object rawMessage: error.getRawMessages()) {
+errors.add((byte[]) rawMessage);
+ }
+ if (error.getThrowable().isPresent()) {
+Throwable throwable = error.getThrowable().get();
+LOG.error("Error parsing message: " + throwable.getMessage(),
throwable);
+ }
+
+}
+
@SuppressWarnings("unchecked")
public ProcessorResult> getResults() {
return new
ProcessorResult.Builder>().withProcessErrors(errors)
[metron] branch master updated: METRON-2109 Add option to use Metron GUID as the id in Elasticsearch (merrimanr) closes apache/metron#1403
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 373ac51 METRON-2109 Add option to use Metron GUID as the id in
Elasticsearch (merrimanr) closes apache/metron#1403
373ac51 is described below
commit 373ac513b0779c9399fce299094560a0261a6590
Author: merrimanr
AuthorDate: Wed May 22 14:30:48 2019 -0500
METRON-2109 Add option to use Metron GUID as the id in Elasticsearch
(merrimanr) closes apache/metron#1403
---
metron-platform/metron-common/README.md| 1 +
.../configuration/IndexingConfigurations.java | 17 +
.../writer/IndexingWriterConfiguration.java| 5 ++
.../configuration/writer/WriterConfiguration.java | 9 +++
.../configuration/IndexingConfigurationsTest.java | 82 ++
.../bulk/ElasticsearchBulkDocumentWriter.java | 2 +-
.../elasticsearch/writer/ElasticsearchWriter.java | 13 ++--
.../ElasticsearchSearchIntegrationTest.java| 6 +-
.../components/ElasticSearchComponent.java | 18 +++--
.../writer/ElasticsearchWriterTest.java| 62 +++-
metron-platform/metron-indexing/README.md | 18 -
11 files changed, 220 insertions(+), 13 deletions(-)
diff --git a/metron-platform/metron-common/README.md
b/metron-platform/metron-common/README.md
index 5144be7..4d19769 100644
--- a/metron-platform/metron-common/README.md
+++ b/metron-platform/metron-common/README.md
@@ -87,6 +87,7 @@ but a convenient index is provided here:
| [`es.port`](../metron-elasticsearch#esport)
| Indexing | String | N/A
|
| [`es.date.format`](../metron-elasticsearch#esdateformat)
| Indexing | String |
`es_date_format`|
| [`es.client.settings`](../metron-elasticsearch#esclientsettings)
| Indexing | Object | N/A
|
+|
[`indexing.writer.elasticsearch.setDocumentId`](../metron-indexing#elasticsearch)
| Indexing |
Boolean| N/A |
| [`solr.zookeeper`](../metron-solr#configuration)
| Indexing | String |
`solr_zookeeper_url`|
| [`solr.commitPerBatch`](../metron-solr#configuration)
| Indexing | String | N/A
|
| [`solr.commit.soft`](../metron-solr#configuration)
| Indexing | String | N/A
|
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java
index 584bed1..5001767 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java
@@ -36,6 +36,8 @@ public class IndexingConfigurations extends Configurations {
public static final String INDEX_CONF = "index";
public static final String OUTPUT_PATH_FUNCTION_CONF = "outputPathFunction";
public static final String FIELD_NAME_CONVERTER_CONF = "fieldNameConverter";
+ public static final String SET_DOCUMENT_ID_CONF = "setDocumentId";
+ public static final String GLOBAL_ELASTICSEARCH_SET_DOCUMENT_ID_CONF =
"indexing.writer.elasticsearch.setDocumentId";
/**
* Gets the indexing config for a specific sensor.
@@ -184,6 +186,10 @@ public class IndexingConfigurations extends Configurations
{
return getFieldNameConverter(getSensorIndexingConfig(sensorName,
writerName), sensorName);
}
+ public boolean isSetDocumentId(String sensorName, String writerName) {
+return isSetDocumentId(getGlobalConfig(true),
getSensorIndexingConfig(sensorName, writerName));
+ }
+
/**
* Retrieves the enabled value from the config.
*
@@ -268,6 +274,17 @@ public class IndexingConfigurations extends Configurations
{
}
/**
+ * Determines if the Metron generated id should be used when indexing
+ *
+ * @param globalConf The global config
+ * @param sensorConf The indexing config for a given sensor
+ * @return True if the Metron generated id should be used as the id, False
otherwise
+ */
+ public static boole
[metron] branch master updated: METRON-2106 Escalation topic setting in Ambari has no effect (merrimanr) closes apache/metron#1400
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 57bbab2 METRON-2106 Escalation topic setting in Ambari has no effect
(merrimanr) closes apache/metron#1400
57bbab2 is described below
commit 57bbab2c899877ddb114082bff111444b3089b00
Author: merrimanr
AuthorDate: Mon May 13 15:57:13 2019 -0500
METRON-2106 Escalation topic setting in Ambari has no effect (merrimanr)
closes apache/metron#1400
---
metron-interface/metron-rest/src/main/scripts/metron-rest.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
index 6f07925..19d0a33 100644
--- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
+++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
@@ -83,7 +83,7 @@ echo
"METRON_SPRING_PROFILES_ACTIVE=${METRON_SPRING_PROFILES_ACTIVE}"
# the vagrant Spring profile provides configuration values, otherwise
configuration is provided by rest_application.yml
if [[ !(${METRON_SPRING_PROFILES_ACTIVE} == *"vagrant"*) ]]; then
-METRON_CONFIG_LOCATION="
--spring.config.location=$METRON_HOME/config/rest_application.yml,classpath:/application.yml"
+METRON_CONFIG_LOCATION="
--spring.config.additional-location=file:$METRON_HOME/config/rest_application.yml"
echo "METRON_CONFIG_LOCATION=${METRON_CONFIG_LOCATION}"
METRON_SPRING_OPTIONS+=${METRON_CONFIG_LOCATION}
fi
[metron] branch master updated: METRON-2090 Full dev is failing with missing org.mortbay.jetty:jetty-util:jar:6.1.26.hwx dependency (merrimanr) closes apache/metron#1391
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new f99cc54 METRON-2090 Full dev is failing with missing
org.mortbay.jetty:jetty-util:jar:6.1.26.hwx dependency (merrimanr) closes
apache/metron#1391
f99cc54 is described below
commit f99cc54d367c1f36433f7a41e1c89e7bf75786fb
Author: merrimanr
AuthorDate: Thu Apr 25 11:34:29 2019 -0500
METRON-2090 Full dev is failing with missing
org.mortbay.jetty:jetty-util:jar:6.1.26.hwx dependency (merrimanr) closes
apache/metron#1391
---
.../metron-writer/metron-writer-storm/pom.xml | 106 +
1 file changed, 67 insertions(+), 39 deletions(-)
diff --git a/metron-platform/metron-writer/metron-writer-storm/pom.xml
b/metron-platform/metron-writer/metron-writer-storm/pom.xml
index 5e0ec56..1f4c1dd 100644
--- a/metron-platform/metron-writer/metron-writer-storm/pom.xml
+++ b/metron-platform/metron-writer/metron-writer-storm/pom.xml
@@ -30,45 +30,65 @@
1.10
-
-org.apache.metron
-metron-common-storm
-${project.parent.version}
-
-
-org.apache.metron
-metron-writer-common
-${project.parent.version}
-
-
-org.apache.storm
-storm-core
-${global_storm_version}
-provided
-
-
-org.apache.logging.log4j
-log4j-core
-
-
-org.apache.logging.log4j
-log4j-api
-
-
-servlet-api
-javax.servlet
-
-
-log4j-over-slf4j
-org.slf4j
-
-
-log4j-slf4j-impl
-org.apache.logging.log4j
-
-
-
-
+
+org.apache.metron
+metron-common-storm
+${project.parent.version}
+
+
+org.apache.metron
+metron-writer-common
+${project.parent.version}
+
+
+org.apache.hadoop
+hadoop-common
+${global_hadoop_version}
+
+
+servlet-api
+javax.servlet
+
+
+commons-httpclient
+commons-httpclient
+
+
+org.slf4j
+slf4j-log4j12
+
+
+provided
+
+
+org.apache.storm
+storm-core
+${global_storm_version}
+provided
+
+
+org.apache.logging.log4j
+log4j-core
+
+
+org.apache.logging.log4j
+log4j-api
+
+
+servlet-api
+javax.servlet
+
+
+log4j-over-slf4j
+org.slf4j
+
+
+log4j-slf4j-impl
+org.apache.logging.log4j
+
+
+
+
org.apache.storm
storm-hdfs
${global_storm_version}
@@ -81,6 +101,14 @@
org.apache.hadoop
hadoop-client
+
+org.mortbay.jetty
+jetty
+
+
+org.mortbay.jetty
+jetty-util
+
[metron] branch master updated: METRON-2091 SimpleHBaseEnrichmentWriterTest should be included in tests (merrimanr) closes apache/metron#1392
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new fd6bbcc METRON-2091 SimpleHBaseEnrichmentWriterTest should be
included in tests (merrimanr) closes apache/metron#1392
fd6bbcc is described below
commit fd6bbcc4ab9862ce0891bd7570a7654dcd68ea46
Author: merrimanr
AuthorDate: Wed Apr 24 13:16:18 2019 -0500
METRON-2091 SimpleHBaseEnrichmentWriterTest should be included in tests
(merrimanr) closes apache/metron#1392
---
metron-platform/metron-writer/metron-writer-common/pom.xml | 7 +++
.../metron/writer/hbase/SimpleHBaseEnrichmentWriterTest.java | 0
2 files changed, 7 insertions(+)
diff --git a/metron-platform/metron-writer/metron-writer-common/pom.xml
b/metron-platform/metron-writer/metron-writer-common/pom.xml
index ef4fe0e..c4e2799 100644
--- a/metron-platform/metron-writer/metron-writer-common/pom.xml
+++ b/metron-platform/metron-writer/metron-writer-common/pom.xml
@@ -190,6 +190,13 @@
stellar-common
${project.parent.version}
+
+org.apache.metron
+metron-hbase
+${project.parent.version}
+test-jar
+test
+
diff --git
a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hbase/SimpleHBaseEnrichmentWriterTest.java
b/metron-platform/metron-writer/metron-writer-common/src/test/java/org/apache/metron/writer/hbase/SimpleHBaseEnrichmentWriterTest.java
similarity index 100%
rename from
metron-platform/metron-writer/src/test/java/org/apache/metron/writer/hbase/SimpleHBaseEnrichmentWriterTest.java
rename to
metron-platform/metron-writer/metron-writer-common/src/test/java/org/apache/metron/writer/hbase/SimpleHBaseEnrichmentWriterTest.java
[metron] branch master updated: METRON-2078 Remove Storm dependency from metron-writer (merrimanr) closes apache/metron#1386
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 51d1c81 METRON-2078 Remove Storm dependency from metron-writer
(merrimanr) closes apache/metron#1386
51d1c81 is described below
commit 51d1c812c1e45f57da8c27fe37fd13797707884e
Author: merrimanr
AuthorDate: Fri Apr 19 14:11:12 2019 -0500
METRON-2078 Remove Storm dependency from metron-writer (merrimanr) closes
apache/metron#1386
---
metron-analytics/metron-profiler-storm/pom.xml | 2 +-
.../metron/storm/common/utils/ErrorUtilsTest.java | 42 ---
.../org/apache/metron/common/utils/ErrorUtils.java | 113 ---
.../apache/metron/common/utils/RuntimeErrors.java | 79 +
.../metron/common/utils/RuntimeErrorsTest.java}| 35 +-
.../dataloads/nonbulk/taxii/TaxiiHandler.java | 4 +-
metron-platform/metron-elasticsearch/pom.xml | 2 +-
.../metron-enrichment-storm/pom.xml| 2 +-
metron-platform/metron-indexing/pom.xml| 41 +--
.../integration/HDFSIndexingIntegrationTest.java | 3 +-
metron-platform/metron-management/pom.xml | 4 +-
.../metron-parsing/metron-parsers-common/pom.xml | 2 +-
.../metron-parsing/metron-parsing-storm/pom.xml| 6 +
.../{ => metron-writer-common}/pom.xml | 67 +---
.../org/apache/metron/writer/AbstractWriter.java | 0
.../org/apache/metron/writer/BatchSizePolicy.java | 0
.../apache/metron/writer/BatchTimeoutPolicy.java | 0
.../apache/metron/writer/BulkWriterComponent.java | 0
.../java/org/apache/metron/writer/FlushPolicy.java | 0
.../java/org/apache/metron/writer/NoopWriter.java | 0
.../apache/metron/writer/WriterToBulkWriter.java | 0
.../writer/hbase/SimpleHbaseEnrichmentWriter.java | 0
.../apache/metron/writer/kafka/KafkaWriter.java| 0
.../src/main/resources/META-INF/LICENSE| 0
.../src/main/resources/META-INF/NOTICE | 0
.../apache/metron/writer/BatchSizePolicyTest.java | 0
.../metron/writer/BatchTimeoutPolicyTest.java | 0
.../metron/writer/BulkWriterComponentTest.java | 4 +-
.../org/apache/metron/writer/NoopWriterTest.java | 0
.../metron/writer/kafka/KafkaWriterTest.java | 0
.../src/test/resources/log4j.properties| 0
.../metron-writer/metron-writer-storm/pom.xml | 93 ++
.../org/apache/metron/writer/AckTuplesPolicy.java | 0
.../metron/writer/bolt/BatchTimeoutHelper.java | 0
.../metron/writer/bolt/BulkMessageWriterBolt.java | 0
.../writer/hdfs/ClonedSyncPolicyCreator.java | 0
.../org/apache/metron/writer/hdfs/HdfsWriter.java | 0
.../writer/hdfs/PathExtensionFileNameFormat.java | 0
.../metron/writer/hdfs/SourceAwareMoveAction.java | 0
.../apache/metron/writer/hdfs/SourceHandler.java | 0
.../metron/writer/hdfs/SourceHandlerCallback.java | 0
.../metron/writer/hdfs/SourceHandlerKey.java | 0
.../metron/writer/hdfs/SyncPolicyCreator.java | 0
.../apache/metron/writer/AckTuplesPolicyTest.java | 17 +-
.../metron/writer/bolt/BatchTimeoutHelperTest.java | 0
.../writer/bolt/BulkMessageWriterBoltTest.java | 52 +--
.../writer/hdfs/ClonedSyncPolicyCreatorTest.java | 0
.../apache/metron/writer/hdfs/HdfsWriterTest.java | 0
.../hdfs/PathExtensionFileNameFormatTest.java | 0
.../metron/writer/hdfs/SourceHandlerTest.java | 0
metron-platform/metron-writer/pom.xml | 366 ++---
51 files changed, 270 insertions(+), 664 deletions(-)
diff --git a/metron-analytics/metron-profiler-storm/pom.xml
b/metron-analytics/metron-profiler-storm/pom.xml
index 66deb4d..fe615e2 100644
--- a/metron-analytics/metron-profiler-storm/pom.xml
+++ b/metron-analytics/metron-profiler-storm/pom.xml
@@ -84,7 +84,7 @@
org.apache.metron
-metron-writer
+metron-writer-storm
${project.parent.version}
diff --git
a/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/utils/ErrorUtilsTest.java
b/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/utils/ErrorUtilsTest.java
index 037a06c..7d0694e 100644
---
a/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/utils/ErrorUtilsTest.java
+++
b/metron-platform/metron-common-streaming/metron-common-storm/src/test/java/org/apache/metron/storm/common/utils/ErrorUtilsTest.java
@@ -19,17 +19,10 @@ package org.apache.metron.storm.common.utils;
import org.apache.metron.common.Constants;
import org.apache.metron.common.error.MetronError;
-import org.apache.metron.common.utils.ErrorUtils;
import org.apache.metron.test.error.MetronErrorJSONMatc
[metron] branch master updated: METRON-2065 Setting Parser Output Topic in Sensor Config is broken (merrimanr) closes apache/metron#1377
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new c453b1e METRON-2065 Setting Parser Output Topic in Sensor Config is
broken (merrimanr) closes apache/metron#1377
c453b1e is described below
commit c453b1edf27a4a2883e4c055246d939218908ebb
Author: merrimanr
AuthorDate: Thu Apr 18 08:19:12 2019 -0500
METRON-2065 Setting Parser Output Topic in Sensor Config is broken
(merrimanr) closes apache/metron#1377
---
metron-platform/metron-parsing/README.md | 14 ++
.../parsers/topology/ParserTopologyBuilder.java| 23 ++-
.../topology/ParserTopologyBuilderTest.java| 169 +
3 files changed, 202 insertions(+), 4 deletions(-)
diff --git a/metron-platform/metron-parsing/README.md
b/metron-platform/metron-parsing/README.md
index e5368fe..085fb33 100644
--- a/metron-platform/metron-parsing/README.md
+++ b/metron-platform/metron-parsing/README.md
@@ -158,6 +158,19 @@ There are two general types types of parsers:
```
"timestamp":"TO_EPOCH_TIMESTAMP(timestamp_str, timestamp_format,
timezone_name )"
```
+
+## Parser Message Routing
+
+Messages are routed to the Kafka `enrichments` topic by default. The output
topic can be changed with the `output_topic`
+option when [Starting the Parser
Topology](metron-parsing-storm/README.md#starting-the-parser-topology) or with
the `outputTopic`
+[Parser Configuration](#parser-configuration) setting. The order of
precedence from highest to lowest is as follows:
+
+1. Parser start script option
+2. Parser configuration setting
+3. Default `enrichments` topic
+
+A message can also be routed to other locations besides Kafka with the
`writerClassName` [Parser Configuration](#parser-configuration) setting.
+Messages can be routed independently for each sensor type when configured with
[Parser Configuration](#parser-configuration) settings.
## Parser Error Routing
@@ -278,6 +291,7 @@ The document is structured in the following way
}
```
+* `writerClassName` : The class used to write messages after they have been
parsed. Defaults to `org.apache.metron.writer.kafka.KafkaWriter`.
* `sensorTopic` : The kafka topic to that the parser will read messages from.
If the topic is prefixed and suffixed by `/`
then it is assumed to be a regex and will match any topic matching the pattern
(e.g. `/bro.*/` would match `bro_cust0`, `bro_cust1` and `bro_cust2`)
* `readMetadata` : Boolean indicating whether to read metadata or not (The
default is raw message strategy dependent). See below for a discussion about
metadata.
diff --git
a/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
index 9dc7b88..b0f9d7d 100644
---
a/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
+++
b/metron-platform/metron-parsing/metron-parsing-storm/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
@@ -228,7 +228,7 @@ public class ParserTopologyBuilder {
* @param securityProtocol An optional security protocol in use.
* @return
*/
- private static KafkaWriter createKafkaWriter(Optional broker,
+ protected static KafkaWriter createKafkaWriter(Optional broker,
String zkQuorum,
Optional
securityProtocol) {
KafkaWriter writer = new KafkaWriter();
@@ -266,6 +266,21 @@ public class ParserTopologyBuilder {
Optional
securityProtocol,
ParserConfigurations configs,
Optional outputTopic) {
+Map writerConfigs =
createWriterConfigs(zookeeperUrl,
+brokerUrl,
+sensorTypeToParserConfig,
+securityProtocol,
+configs,
+outputTopic);
+return new ParserBolt(zookeeperUrl, new ParserRunnerImpl(new
HashSet<>(sensorTypeToParserConfig.keySet())), writerConfigs);
+ }
+
+ protected static Map createWriterConfigs(String
zookeeperUrl,
+
Optional brokerUrl,
+ Map sensorTypeToParserConfig,
+
Optional securityProtocol,
+
ParserConfigurations configs,
+
[metron] branch master updated: METRON-2014 Add architectural documentation for metron-writer (merrimanr) closes apache/metron#1381
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 3a61933 METRON-2014 Add architectural documentation for metron-writer
(merrimanr) closes apache/metron#1381
3a61933 is described below
commit 3a61933ed4fb6ec00286f4c13a22c372adf8d0dc
Author: merrimanr
AuthorDate: Tue Apr 16 13:16:20 2019 -0500
METRON-2014 Add architectural documentation for metron-writer (merrimanr)
closes apache/metron#1381
---
metron-platform/metron-writer/README.md | 89 +
1 file changed, 89 insertions(+)
diff --git a/metron-platform/metron-writer/README.md
b/metron-platform/metron-writer/README.md
index ed4f053..3cc9c87 100644
--- a/metron-platform/metron-writer/README.md
+++ b/metron-platform/metron-writer/README.md
@@ -20,6 +20,95 @@ limitations under the License.
## Introduction
The writer module provides some utilties for writing to outside components
from within Storm. This includes managing bulk writing. An implemention is
included for writing to HDFS in this module. Other writers can be found in
their own modules.
+## Bulk Message Writing
+Most external components encourage messages to be written in batches for
performance reasons. The writer module includes an abstraction for doing this
in an efficient manner. This abstraction provides the following features:
+* A high-level `BulkWriterComponent` class that manages a per-sensor cache of
batched messages and flushes when appropriate
+* An extension point for determining when a batch should be flushed
+* An extention point for handling a bulk message write response after a batch
has been flushed
+
+### Flush Policies
+Flushing behavior is controlled by a collection of `FlushPolicy` objects.
They are responsible for 2 things:
+
+1. Determining when a batch should be flushed
+2. Handling a `BulkWriterResponse` after a batch is flushed and messages are
written with a `BulkMessageWriter`
+
+The `FlushPolicy` interface defines methods for handling these
responsiblities:
+
+- `boolean shouldFlush(String sensorType, WriterConfiguration configurations,
List> messages)`
+- `void onFlush(String sensorType, BulkWriterResponse response)`
+
+There are 2 `FlushPolicy` implementations included by default:
+
+- The `BatchSizePolicy` will flush a batch whenever the batch size reaches a
configured value. This configuration value is represented by the `batchSize`
property in either the parser, enrichment or indexing configuration (whichever
is appropriate in the current context).
+- The `BatchTimeoutPolicy` will flush a batch whenever the batch timeout has
elapsed. This configuration value is represented by the `batchTimeout`
property in either the parser, enrichment or indexing configuration (whichever
is appropriate in the current context). A `maxBatchTimeout` is set at creation
time and serves as the ceiling for a batch timeout. In Storm topologies, this
value is set to 1/2 the tuple timeout setting to ensure messages are always
flushed before their tuples t [...]
+
+For example, a configuration that sets the `batchSize` and `batchTimeout` in a
parser topology will look like:
+```
+{
+ "parserClassName": "org.apache.metron.parsers.bro.BasicBroParser",
+ "sensorTopic": "bro",
+ "parserConfig": {
+"batchSize": 5
+"batchTimeout": 2
+ }
+}
+```
+Similarly for the enrichment topology (configured in the [Global
Configuration](../metron-common#global-configuration)):
+```
+{
+ "enrichment.writer.batchSize": "5",
+ "enrichment.writer.batchTimeout": "2",
+ ...
+}
+```
+And finally for the indexing topology:
+```
+{
+ "elasticsearch": {
+"index": "bro",
+"batchSize": 5,
+"batchTimeout": 2,
+"enabled": true
+ },
+ ...
+}
+```
+
+
+Additional policies can be added as needed. For example, an `AckTuplesPolicy`
is added in the Storm bolts to handle acking tuples after a batch is flushed.
+
+### Bulk Writing Workflow
+The `BulkWriterComponent` class collects messages in separate sensor-specific
caches. This class is instantiated and supplied to classes that need to write
messages to external components. A collection of default `FlushPolicy`
implementations
+are created by default with the option of passing in additional `FlushPolicy`
objects as needed.
+
+Batching and writing messages follows this process:
+
+1. A single message is passed to the `BulkWriterComponent.write` method and
stored in the appropriate cache based on the sensor type. A
`BulkMessageWriter` is also supplied to do the actual writing when messages are
flushed.
+2. The collection of `FlushPolicy` implementations are checked and a batch
[metron] branch master updated: METRON-2026 Remove Storm dependency from metron-common (merrimanr) closes apache/metron#1351
This is an automated email from the ASF dual-hosted git repository. rmerriman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron.git The following commit(s) were added to refs/heads/master by this push: new 2f2c802 METRON-2026 Remove Storm dependency from metron-common (merrimanr) closes apache/metron#1351 2f2c802 is described below commit 2f2c80244b34e8182f72e6f4dc98a2940194eaa0 Author: merrimanr AuthorDate: Mon Apr 15 16:40:51 2019 -0500 METRON-2026 Remove Storm dependency from metron-common (merrimanr) closes apache/metron#1351 --- .../metron/profiler/storm/ProfileSplitterBolt.java | 2 +- .../metron-common-storm/pom.xml| 89 ++ .../metron/storm}/common/bolt/ConfiguredBolt.java | 2 +- .../common/bolt/ConfiguredEnrichmentBolt.java | 2 +- .../storm}/common/bolt/ConfiguredIndexingBolt.java | 2 +- .../storm}/common/bolt/ConfiguredParserBolt.java | 2 +- .../storm}/common/bolt/ConfiguredProfilerBolt.java | 2 +- .../storm}/common/message/BytesFromPosition.java | 2 +- .../storm}/common/message/JSONFromField.java | 2 +- .../common/message/JSONFromFieldByReference.java | 2 +- .../storm}/common/message/JSONFromPosition.java| 2 +- .../storm}/common/message/MessageGetStrategy.java | 2 +- .../storm}/common/message/MessageGetters.java | 2 +- .../storm}/common/message/ObjectFromField.java | 2 +- .../common/message/metadata/RawMessageUtil.java} | 65 ++-- .../storm/common/utils/StormErrorUtils.java} | 32 +--- .../storm}/common/bolt/BaseConfiguredBoltTest.java | 2 +- .../common/bolt/ConfiguredEnrichmentBoltTest.java | 16 ++-- .../common/bolt/ConfiguredParserBoltTest.java | 13 ++-- .../storm}/common/message/MessageGettersTest.java | 2 +- .../message/metadata/RawMessageUtilTest.java | 9 ++- .../metron/storm}/common/utils/ErrorUtilsTest.java | 5 +- metron-platform/metron-common-streaming/pom.xml| 33 metron-platform/metron-common/pom.xml | 75 ++ .../apache/metron/common/error/MetronError.java| 4 +- .../common/message/metadata/MetadataUtil.java | 53 - .../common/message/metadata/RawMessageUtil.java| 59 -- .../org/apache/metron/common/utils/ErrorUtils.java | 20 - .../metron/common/writer/BulkMessageWriter.java| 3 +- .../elasticsearch/writer/ElasticsearchWriter.java | 3 +- .../writer/ElasticsearchWriterTest.java| 18 ++--- .../metron/enrichment/bolt/EnrichmentJoinBolt.java | 2 +- .../enrichment/bolt/GenericEnrichmentBolt.java | 10 +-- .../apache/metron/enrichment/bolt/JoinBolt.java| 10 +-- .../apache/metron/enrichment/bolt/SplitBolt.java | 2 +- .../enrichment/bolt/ThreatIntelJoinBolt.java | 2 +- .../enrichment/bolt/UnifiedEnrichmentBolt.java | 12 +-- .../enrichment/bolt/EnrichmentJoinBoltTest.java| 2 +- .../metron/enrichment/bolt/JoinBoltTest.java | 2 +- .../enrichment/bolt/ThreatIntelJoinBoltTest.java | 2 +- .../metron-parsing/metron-parsing-storm/pom.xml| 2 +- .../org/apache/metron/parsers/bolt/ParserBolt.java | 14 ++-- .../org/apache/metron/parsers/bolt/WriterBolt.java | 8 +- .../apache/metron/parsers/bolt/WriterHandler.java | 4 +- .../apache/metron/parsers/bolt/ParserBoltTest.java | 2 +- .../apache/metron/parsers/bolt/WriterBoltTest.java | 8 +- .../integration/validation/StormParserDriver.java | 2 +- .../org/apache/metron/solr/writer/SolrWriter.java | 3 +- .../schema/SchemaValidationIntegrationTest.java| 2 +- .../apache/metron/solr/writer/SolrWriterTest.java | 4 +- metron-platform/metron-writer/pom.xml | 2 +- .../org/apache/metron/writer/AckTuplesPolicy.java | 2 +- .../java/org/apache/metron/writer/NoopWriter.java | 3 +- .../apache/metron/writer/WriterToBulkWriter.java | 3 +- .../metron/writer/bolt/BulkMessageWriterBolt.java | 16 ++-- .../writer/hbase/SimpleHbaseEnrichmentWriter.java | 3 +- .../org/apache/metron/writer/hdfs/HdfsWriter.java | 7 +- .../apache/metron/writer/kafka/KafkaWriter.java| 3 +- .../apache/metron/writer/AckTuplesPolicyTest.java | 2 +- .../metron/writer/BulkWriterComponentTest.java | 1 - .../writer/bolt/BulkMessageWriterBoltTest.java | 12 ++- .../apache/metron/writer/hdfs/HdfsWriterTest.java | 51 - metron-platform/pom.xml| 1 + 63 files changed, 331 insertions(+), 395 deletions(-) diff --git a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java b/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java index ef58ad9..2161910 100644 --- a/metron-analytics/metron-profiler-storm/src/main/java/org/apache/metron/profiler/storm/ProfileSplitterBolt.java +++ b/metron-analytics/metron-profiler-storm/src/main/java/org
[metron] branch master updated: METRON-2064 Metron REST API overwriting global.json values (merrimanr) closes apache/metron#1376
This is an automated email from the ASF dual-hosted git repository. rmerriman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron.git The following commit(s) were added to refs/heads/master by this push: new 8d7ce16 METRON-2064 Metron REST API overwriting global.json values (merrimanr) closes apache/metron#1376 8d7ce16 is described below commit 8d7ce165050619abf3d8be600ba4c8655b6ad0ed Author: merrimanr AuthorDate: Fri Apr 12 09:47:24 2019 -0500 METRON-2064 Metron REST API overwriting global.json values (merrimanr) closes apache/metron#1376 --- metron-platform/metron-common/README.md | 59 +++-- metron-platform/metron-solr/README.md | 4 +-- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/metron-platform/metron-common/README.md b/metron-platform/metron-common/README.md index cbea9dd..40f1289 100644 --- a/metron-platform/metron-common/README.md +++ b/metron-platform/metron-common/README.md @@ -80,30 +80,41 @@ This configuration is stored in zookeeper, but looks something like Various parts of our stack uses the global config are documented throughout the Metron documentation, but a convenient index is provided here: -| Property Name | Subsystem | Type | Ambari Property| -|-|---||| -| [`es.clustername`](../metron-elasticsearch#esclustername) | Indexing | String | `es_cluster_name` | -| [`es.ip`](../metron-elasticsearch#esip) | Indexing | String | `es_hosts` | -| [`es.port`](../metron-elasticsearch#esport) | Indexing | String | `es_port` | -| [`es.date.format`](../metron-elasticsearch#esdateformat) | Indexing | String | `es_date_format` | -| [`es.client.settings`](../metron-elasticsearch#esclientsettings) | Indexing | Object | N/A | -| [`fieldValidations`](#validation-framework) | Parsing | Object | N/A | -| [`parser.error.topic`](../metron-parsers#parsererrortopic) | Parsing | String | N/A | -| [`stellar.function.paths`](../../metron-stellar/stellar-common#stellarfunctionpaths) | Stellar | CSV String | N/A | -| [`stellar.function.resolver.includes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A| -| [`stellar.function.resolver.excludes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A| -| [`profiler.period.duration`](../../metron-analytics/metron-profiler#profilerperiodduration) | Profiler | Integer| `profiler_period_duration` | -| [`profiler.period.duration.units`](../../metron-analytics/metron-profiler#profilerperioddurationunits) | Profiler | String | `profiler_period_units`| -| [`profiler.writer.batchSize`](../../metron-analytics/metron-profiler/#profilerwriterbatchsize) | Profiler | Integer| N/A | -| [`profiler.writer.batchTimeout`](../../metron-analytics/metron-profiler/#profilerwriterbatchtimeout) | Profiler | Integer| N/A | -| [`update.hbase.table`](../metron-indexing#updatehbasetable) | REST/Indexing | String | `update_hbase_table` | -| [`update.hbase.cf`](../metron-indexing#updatehbasecf) | REST/Indexing | String | `update_hbase_cf` | -| [`geo.hdfs.file`](../metron-enrichment#geohdfsfile) | Enrichment| String | `geo_hdfs_file`| -| [`enrichment.writer.batchSize`](../metron-enrichment#enrichmentwriterbatchsize) | Enrichment| Integer| N/A | -| [`enrichment.writer.batchTimeout`](../metron-enrichment#enrichmentwriterbatchtimeout
[metron] branch master updated: METRON-1654 findOne request after an alert patch returns with the original state of the alert item (merrimanr) closes apache/metron#1344
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 64b0790 METRON-1654 findOne request after an alert patch returns with
the original state of the alert item (merrimanr) closes apache/metron#1344
64b0790 is described below
commit 64b079044f6840fc0bcb1bf9cb665790f37900f4
Author: merrimanr
AuthorDate: Wed Apr 10 10:58:04 2019 -0500
METRON-1654 findOne request after an alert patch returns with the original
state of the alert item (merrimanr) closes apache/metron#1344
---
.../alert-details/alert-details.component.spec.ts | 36 +--
.../alert-details/alert-details.component.ts | 53 ++
.../alerts/alerts-list/alerts-list.component.ts| 31 ++---
.../alerts-list/table-view/table-view.component.ts | 8
.../alerts-list/tree-view/tree-view.component.ts | 24 +-
.../src/app/service/meta-alert.service.ts | 25 +-
.../src/app/service/update.service.ts | 26 ++-
7 files changed, 108 insertions(+), 95 deletions(-)
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.spec.ts
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.spec.ts
index b875613..7800517 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.spec.ts
@@ -35,10 +35,14 @@ import { By } from '@angular/platform-browser';
import { AlertComment } from './alert-comment';
import { Subject } from 'rxjs';
import { ConfirmationType } from 'app/model/confirmation-type';
+import {CommentAddRemoveRequest} from "../../model/comment-add-remove-request";
+import {AlertSource} from "../../model/alert-source";
+import {of} from "rxjs/index";
describe('AlertDetailsComponent', () => {
let component: AlertDetailsComponent;
let fixture: ComponentFixture;
+ let updateService: UpdateService;
beforeEach(async(() => {
TestBed.configureTestingModule({
@@ -83,14 +87,26 @@ describe('AlertDetailsComponent', () => {
beforeEach(() => {
fixture = TestBed.createComponent(AlertDetailsComponent);
component = fixture.componentInstance;
+updateService = fixture.debugElement.injector.get(UpdateService);
fixture.detectChanges();
});
it('should delete a comment.', fakeAsync(() => {
+const responseMock = new AlertSource();
+responseMock.guid = 'guid';
+const removeCommentSpy = spyOn(updateService,
'removeComment').and.returnValue(
+of(responseMock)
+);
+const setAlertSpy = spyOn(component, 'setAlert');
+
expect(component).toBeTruthy();
+component.alertSource = new AlertSource();
+component.alertSource.guid = 'guid';
+component.alertSourceType = 'sourceType';
+const now = Date.now();
component.alertCommentsWrapper = [
new AlertCommentWrapper(
-new AlertComment('lorem ipsum', 'user', Date.now()),
+new AlertComment('lorem ipsum', 'user', now),
(new Date()).toString()
)
];
@@ -101,8 +117,20 @@ describe('AlertDetailsComponent', () => {
deleteComment.nativeElement.click();
tick(500);
fixture.detectChanges();
-expect(component.alertCommentsWrapper.length).toEqual(0);
-const comments =
fixture.debugElement.queryAll(By.css('[data-qe-id="comment"]'));
-expect(comments.length).toEqual(0);
+
+const expectedCommentRequest = new CommentAddRemoveRequest();
+expectedCommentRequest.guid = 'guid';
+expectedCommentRequest.comment = 'lorem ipsum';
+expectedCommentRequest.username = 'user';
+expectedCommentRequest.sensorType = 'sourceType';
+expectedCommentRequest.timestamp = now;
+
+const expectedAlertSource = new AlertSource();
+expectedAlertSource.guid = 'guid';
+
+expect(removeCommentSpy).toHaveBeenCalledWith(expectedCommentRequest);
+expect(removeCommentSpy).toHaveBeenCalledTimes(1);
+expect(setAlertSpy).toHaveBeenCalledWith(expectedAlertSource);
+expect(setAlertSpy).toHaveBeenCalledTimes(1);
}));
});
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index 73458a6..f5c55fa 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -98,7 +98,6 @@ export class AlertDetailsComponent implements OnInit {
this.alertCommentStr = '';
this.searchService.getAlert(this.alertSourceType,
this.alertId).s
[metron] branch master updated: METRON-2022 Metron rest creates large number of connections to ZK which causes subsequent connection to zk fail (merrimanr) closes apache/metron#1367
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new a1c257c METRON-2022 Metron rest creates large number of connections
to ZK which causes subsequent connection to zk fail (merrimanr) closes
apache/metron#1367
a1c257c is described below
commit a1c257c6a583ba705d5e551fb5e7f5c9862097ac
Author: merrimanr
AuthorDate: Mon Apr 8 12:25:00 2019 -0500
METRON-2022 Metron rest creates large number of connections to ZK which
causes subsequent connection to zk fail (merrimanr) closes apache/metron#1367
---
.../metron/solr/client/SolrClientFactory.java | 84 ++
.../java/org/apache/metron/solr/dao/SolrDao.java | 33 +
.../apache/metron/solr/dao/SolrMetaAlertDao.java | 6 +-
.../solr/dao/SolrMetaAlertRetrieveLatestDao.java | 8 ++-
.../metron/solr/dao/SolrMetaAlertUpdateDao.java| 5 +-
.../metron/solr/client/SolrClientFactoryTest.java | 57 +++
.../org/apache/metron/solr/dao/SolrDaoTest.java| 50 ++---
.../metron/solr/dao/SolrMetaAlertDaoTest.java | 20 ++
.../integration/SolrMetaAlertIntegrationTest.java | 11 +--
.../SolrRetrieveLatestIntegrationTest.java | 2 +
.../integration/SolrSearchIntegrationTest.java | 2 +
.../integration/SolrUpdateIntegrationTest.java | 2 +
12 files changed, 194 insertions(+), 86 deletions(-)
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/client/SolrClientFactory.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/client/SolrClientFactory.java
new file mode 100644
index 000..4c704d9
--- /dev/null
+++
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/client/SolrClientFactory.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.solr.client;
+
+import com.google.common.base.Splitter;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.metron.solr.SolrConstants.SOLR_ZOOKEEPER;
+
+/**
+ * Factory for creating a SolrClient. The default implementation of
SolrClient is CloudSolrClient.
+ */
+public class SolrClientFactory {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static SolrClient solrClient;
+
+ /**
+ * Creates a SolrClient.
+ * @param globalConfig Global config
+ * @return SolrClient
+ */
+ public static SolrClient create(Map globalConfig) {
+if (solrClient == null) {
+ synchronized (SolrClientFactory.class) {
+if (solrClient == null) {
+ solrClient = new
CloudSolrClient.Builder().withZkHost(getZkHosts(globalConfig)).build();
+}
+ }
+}
+return solrClient;
+ }
+
+ /**
+ * Closes the SolrClient connection and releases the reference.
+ */
+ public static void close() {
+synchronized (SolrClientFactory.class) {
+ if (solrClient != null) {
+try {
+ solrClient.close();
+} catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+} finally {
+ solrClient = null;
+}
+ }
+}
+ }
+
+ /**
+ * Retrieves zookeeper hosts from the global config and formats them for
CloudSolrClient instantiation.
+ * @param globalConfig Global config
+ * @return A list of properly formatted zookeeper servers
+ */
+ protected static List getZkHosts(Map globalConfig) {
+return Splitter.on(',').trimResults()
+.splitToList((String) globalConfig.getOrDefault(SOLR_ZOOKEEPER,
""));
+ }
+}
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrDao.java
index e6906f3..4a58808 100644
---
a/metron-platform
[metron] branch master updated: METRON-2039 Time range queries do not work with Solr (merrimanr) closes apache/metron#1359
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 4da8990 METRON-2039 Time range queries do not work with Solr
(merrimanr) closes apache/metron#1359
4da8990 is described below
commit 4da899043ffdb222f8a928cd30bc0f47b99c632a
Author: merrimanr
AuthorDate: Tue Apr 2 11:50:36 2019 -0500
METRON-2039 Time range queries do not work with Solr (merrimanr) closes
apache/metron#1359
---
.../metron-alerts/src/app/model/filter.spec.ts | 70 +++
.../metron-alerts/src/app/model/filter.ts | 2 +-
.../shared/time-range/time-range.component.spec.ts | 80 +-
.../app/shared/time-range/time-range.component.ts | 6 +-
.../metron-alerts/src/app/utils/utils.spec.ts | 76
.../metron-alerts/src/app/utils/utils.ts | 6 +-
6 files changed, 232 insertions(+), 8 deletions(-)
diff --git a/metron-interface/metron-alerts/src/app/model/filter.spec.ts
b/metron-interface/metron-alerts/src/app/model/filter.spec.ts
new file mode 100644
index 000..d073ebd
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/model/filter.spec.ts
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { Filter } from './filter';
+import { TIMESTAMP_FIELD_NAME } from 'app/utils/constants';
+import { Utils } from 'app/utils/utils';
+
+describe('model.Filter', () => {
+
+ const expectedTimeRangeQueryString = '(timestamp:[155286360 TO
155295000] OR ' +
+ 'metron_alert.timestamp:[155286360 TO 155295000])';
+
+ it('should have getQueryString function', () => {
+const filter = new Filter('testField', 'someValue', false);
+expect(typeof filter.getQueryString).toBe('function');
+ });
+
+ it('getQueryString for basic filter', () => {
+const filter = new Filter('testField', 'someValue', false);
+expect(filter.getQueryString()).toBe('(testField:someValue OR
metron_alert.testField:someValue)');
+ });
+
+ it('getQueryString for guid filter', () => {
+const filter = new Filter('guid', 'someValue', false);
+expect(filter.getQueryString()).toBe('(guid:"someValue" OR
metron_alert.guid:"someValue")');
+ });
+
+ it('getQueryString for time range filter with []', () => {
+const filter = new Filter(TIMESTAMP_FIELD_NAME, '[155286360 TO
155295000]', false);
+expect(filter.getQueryString()).toBe(expectedTimeRangeQueryString);
+ });
+
+ it('getQueryString for time range filter without []', () => {
+const filter = new Filter(TIMESTAMP_FIELD_NAME, '155286360 TO
155295000', false);
+expect(filter.getQueryString()).toBe(expectedTimeRangeQueryString);
+ });
+
+ it('getQueryString for time range filter for display', () => {
+const filter = new Filter(TIMESTAMP_FIELD_NAME, '[155286360 TO
155295000]', true);
+expect(filter.getQueryString()).toBe('(timestamp:\\[155286360\\ TO\\
155295000\\] OR ' +
+ 'metron_alert.timestamp:\\[155286360\\ TO\\ 155295000\\])');
+ });
+
+ /**
+ * Actual time range and quick range conversion tested in utils/utils.spec.ts
+ * until further refactoring.
+ */
+ it('getQueryString for time range filter should call
Utils.timeRangeToDateObj', () => {
+const timeRange = '155286360 TO 155295000';
+spyOn(Utils, 'timeRangeToDateObj').and.callThrough();
+const filter = new Filter(TIMESTAMP_FIELD_NAME, timeRange, false);
+
+filter.getQueryString();
+expect(Utils.timeRangeToDateObj).toHaveBeenCalledWith(timeRange);
+ });
+});
diff --git a/metron-interface/metron-alerts/src/app/model/filter.ts
b/metron-interface/metron-alerts/src/app/model/filter.ts
index c90c7a6..5d56c49 100644
--- a/metron-interface/metron-alerts/src/app/model/filter.ts
+++ b/metron-interface/metron-alerts/src/app/model/filter.ts
@@ -51,7 +51,7 @@ export class Filter {
this.dateFilterValue = Utils.timeRangeToDateObj(this.value);
if (this.dateFilterValue !==
[metron] branch master updated: METRON-2019 Improve Metron REST Logging (merrimanr) closes apache/metron#1347
This is an automated email from the ASF dual-hosted git repository. rmerriman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron.git The following commit(s) were added to refs/heads/master by this push: new 34a1fd5 METRON-2019 Improve Metron REST Logging (merrimanr) closes apache/metron#1347 34a1fd5 is described below commit 34a1fd513c8f27b130b6ba8edbb3d4a1ed463993 Author: merrimanr AuthorDate: Tue Mar 5 17:24:18 2019 -0600 METRON-2019 Improve Metron REST Logging (merrimanr) closes apache/metron#1347 --- dependencies_with_url.csv | 1 + metron-interface/metron-rest/README.md | 14 metron-interface/metron-rest/pom.xml | 16 ++-- .../apache/metron/rest/MetronExceptionHandler.java | 48 .../apache/metron/rest/MetronRestApplication.java | 3 - .../apache/metron/rest/config/LoggingConfig.java | 39 ++ .../rest/web/filter/ResponseLoggingFilter.java | 87 ++ .../src/main/resources/log4j.properties| 16 .../src/test/resources/log4j.properties| 19 - 9 files changed, 197 insertions(+), 46 deletions(-) diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index e16650d..0ae7ab4 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -385,6 +385,7 @@ org.springframework.boot:spring-boot-starter-jdbc:jar:2.0.1.RELEASE:compile,ASLv org.springframework.boot:spring-boot-starter-json:jar:2.0.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot org.springframework.boot:spring-boot-starter-logging:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot org.springframework.boot:spring-boot-starter-logging:jar:2.0.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot +org.springframework.boot:spring-boot-starter-log4j2:jar:2.0.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot org.springframework.boot:spring-boot-starter-security:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot org.springframework.boot:spring-boot-starter-security:jar:2.0.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot org.springframework.boot:spring-boot-starter-tomcat:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 11f06bf..8d7d99f 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -248,6 +248,20 @@ Setting active profiles is done with the METRON_SPRING_PROFILES_ACTIVE variable. METRON_SPRING_PROFILES_ACTIVE="vagrant,dev" ``` +## Logging + +Logging for the REST application can be configured in Ambari. Log levels can be changed at the root, package and class level: + +1. Navigate to Services > Metron > Configs > REST and locate the `Metron Spring options` setting. + +1. Logging configuration is exposed through Spring properties as explained [here](https://docs.spring.io/spring-boot/docs/current/reference/html/howto-logging.html#howto-logging). + +1. The root logging level defaults to ERROR but can be changed to INFO by adding `--logging.level.root=INFO` to the `Metron Spring options` setting. + +1. The Metron REST logging level can be changed to INFO by adding `--logging.level.org.apache.metron.rest=INFO`. + +1. HTTP request and response logging can be enabled by adding `--logging.level.org.springframework.web.filter.CommonsRequestLoggingFilter=DEBUG --logging.level.org.apache.metron.rest.web.filter.ResponseLoggingFilter=DEBUG`. + ## Pcap Query The REST application exposes endpoints for querying Pcap data. For more information about filtering options see [Query Filter Utility](../../metron-platform/metron-pcap-backend#query-filter-utility). diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml index 01f1aed..cc58bf8 100644 --- a/metron-interface/metron-rest/pom.xml +++ b/metron-interface/metron-rest/pom.xml @@ -78,14 +78,10 @@ org.springframework.boot spring-boot-starter-web - -ch.qos.logback -logback-classic - - -org.slf4j -log4j-over-slf4j - + +org.springframework.boot +spring-boot-starter-logging + org.hibernate hibernate-validator @@ -94,6 +90,10 @@ org.springframework.boot +spring-boot-starter-log4j2 + + +org.springframework.boot spring-boot-starter-security diff --git a/metron-interface/metron-rest/src/main/java/org
[metron] branch master updated: METRON-2016 Parser aggregate groups should be persisted and available through REST (merrimanr) closes apache/metron#1346
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 6d41526 METRON-2016 Parser aggregate groups should be persisted and
available through REST (merrimanr) closes apache/metron#1346
6d41526 is described below
commit 6d41526c3eff511e5fe11368c6f3f0a837507ba1
Author: merrimanr
AuthorDate: Tue Mar 5 17:20:25 2019 -0600
METRON-2016 Parser aggregate groups should be persisted and available
through REST (merrimanr) closes apache/metron#1346
---
.../CURRENT/package/scripts/parser_commands.py | 10 +-
metron-interface/metron-rest/README.md | 33 +++
metron-interface/metron-rest/pom.xml | 6 +
.../controller/SensorParserGroupController.java| 93 +++
.../rest/service/SensorParserGroupService.java | 39 +++
.../service/impl/SensorParserGroupServiceImpl.java | 136 +++
.../rest/service/impl/StormAdminServiceImpl.java | 50 +++-
.../rest/service/impl/StormStatusServiceImpl.java | 68 --
...SensorParserGroupControllerIntegrationTest.java | 266 +
.../impl/SensorParserGroupServiceImplTest.java | 203
.../service/impl/StormAdminServiceImplTest.java| 40 +++-
.../service/impl/StormStatusServiceImplTest.java | 51 +++-
.../common/configuration/ParserConfigurations.java | 17 ++
.../common/configuration/SensorParserGroup.java| 83 +++
.../metron/parsers/topology/ParserTopologyCLI.java | 16 +-
15 files changed, 1063 insertions(+), 48 deletions(-)
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
index 18780d9..a687085 100755
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py
@@ -65,14 +65,20 @@ class ParserCommands:
def __get_aggr_parsers(self, params):
"""
Fetches the list of aggregated (and regular) parsers and returns a
list.
-If the input list of parsers were "bro,snort,yaf", "bro,snort" and
yaf, for example,
-then this method will return ["bro,snort,yaf", "bro,snort", "yaf"]
+If the input list of parsers were "bro,yaf,snort", "bro,snort" and
yaf, for example,
+then this method will return ["bro,snort,yaf", "bro,snort", "yaf"].
Sensors within
+a group are sorted alphabetically.
:param params:
:return: List containing the names of parsers
"""
parserList = []
parsers = shlex.shlex(params.parsers)
for name in parsers:
+sensors = name.strip('",').split(",")
+# if name contains multiple sensors, sort them alphabetically
+if len(sensors) > 1:
+sensors.sort()
+name = '"' + ",".join(sensors) + '"'
parserList.append(name.strip(','))
return [s.translate(None, "'[]") for s in filter(None, parserList)]
diff --git a/metron-interface/metron-rest/README.md
b/metron-interface/metron-rest/README.md
index 8aa172b..11f06bf 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -344,6 +344,10 @@ Request and Response objects are JSON formatted. The JSON
schemas are available
| [ `GET
/api/v1/sensor/parser/config/reload/available`](#get-apiv1sensorparserconfigreloadavailable)|
| [ `DELETE
/api/v1/sensor/parser/config/{name}`](#delete-apiv1sensorparserconfigname)|
| [ `GET
/api/v1/sensor/parser/config/{name}`](#get-apiv1sensorparserconfigname)|
+| [ `POST /api/v1/sensor/parser/group`](#post-apiv1sensorparsergroup)|
+| [ `GET /api/v1/sensor/parser/group/{name}`](#get-apiv1sensorparsergroupname)|
+| [ `GET /api/v1/sensor/parser/group`](#get-apiv1sensorparsergroup)|
+| [ `DELETE
/api/v1/sensor/parser/group/{name}`](#delete-apiv1sensorparsergroupname)|
| [ `POST
/api/v1/stellar/apply/transformations`](#post-apiv1stellarapplytransformations)|
| [ `GET /api/v1/stellar/list`](#get-apiv1stellarlist)|
| [ `GET /api/v1/stellar/list/functions`](#get-apiv1stellarlistfunctions)|
@@ -787,6 +791,35 @@ Request and Response objects are JSON formatted. The JSON
schemas are available
* Returns:
* 200 - Returns SensorParserConfig
* 404 - SensorParserConfig is missing
+
+### `POST /api/v1/sensor/parser/group`
+
[metron] branch master updated: METRON-1998 Only one sensor is flushed by tick tuple (merrimanr) closes apache/metron#1335
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new d275a95 METRON-1998 Only one sensor is flushed by tick tuple
(merrimanr) closes apache/metron#1335
d275a95 is described below
commit d275a958cb5b7bbab533d273dff80bc1e3e65fd0
Author: merrimanr
AuthorDate: Wed Feb 20 16:11:50 2019 -0600
METRON-1998 Only one sensor is flushed by tick tuple (merrimanr) closes
apache/metron#1335
---
.../apache/metron/writer/BulkWriterComponent.java | 13 +++-
.../metron/writer/BulkWriterComponentTest.java | 78 ++
2 files changed, 88 insertions(+), 3 deletions(-)
diff --git
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
index 15e59d3..ad6d4d1 100644
---
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
+++
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
@@ -31,6 +31,8 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.collections.CollectionUtils;
import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.error.MetronError;
@@ -308,7 +310,7 @@ public class BulkWriterComponent {
}
long endTime = System.currentTimeMillis();
long elapsed = endTime - startTime;
-LOG.debug("Bulk batch for sensor {} completed in ~{} ns", sensorType,
elapsed);
+LOG.debug("Flushed batch successfully; sensorType={}, batchSize={},
took={} ms", sensorType, CollectionUtils.size(tupleList), elapsed);
}
// Flushes all queues older than their batchTimeouts.
@@ -320,17 +322,22 @@ public class BulkWriterComponent {
{
// No need to do "all" sensorTypes here, just the ones that have data
batched up.
// Note queues with batchSize == 1 don't get batched, so they never
persist in the sensorTupleMap.
-for (String sensorType : sensorTupleMap.keySet()) {
+// Sensors are removed from the sensorTupleMap when flushed so we need to
iterate over a copy of sensorTupleMap keys
+// to avoid a ConcurrentModificationException.
+for (String sensorType : new HashSet<>(sensorTupleMap.keySet())) {
long[] batchTimeoutInfo = batchTimeoutMap.get(sensorType);
if (batchTimeoutInfo == null //Shouldn't happen, but conservatively
flush if so
|| clock.currentTimeMillis() - batchTimeoutInfo[LAST_CREATE_TIME_MS]
>= batchTimeoutInfo[TIMEOUT_MS]) {
flush(sensorType, bulkMessageWriter, configurations, messageGetStrategy
, sensorTupleMap.get(sensorType),
sensorMessageMap.get(sensorType));
-return;
}
}
}
+ public int getDefaultBatchTimeout() {
+return defaultBatchTimeout;
+ }
+
/**
* @param defaultBatchTimeout
*/
diff --git
a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java
b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java
index 1a05ba4..754a650 100644
---
a/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java
+++
b/metron-platform/metron-writer/src/test/java/org/apache/metron/writer/BulkWriterComponentTest.java
@@ -17,10 +17,13 @@
*/
package org.apache.metron.writer;
+import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.argThat;
import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
@@ -38,6 +41,7 @@ import
org.apache.metron.common.configuration.writer.ParserWriterConfiguration;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
import org.apache.metron.common.error.MetronError;
import org.apache.metron.common.message.MessageGetStrategy;
+import org.apache.metron.common.system.Clock;
import org.apache.metron.common.utils.ErrorUtils;
import org.apache.metron.common.writer.BulkMessageWriter;
import org.apache.metron.common.writer.BulkWriterResponse;
@@ -270,4 +274,78 @@ public class BulkWriterComponentTest {
verify(collector, times(1)).ack(missingTuple);
verifyNoMoreInteractions(collector);
}
+
+ @Test
+ public void flushTimeoutsShouldFlushAllMessagesAfterDefaultTimeout() throws
Exception {
+Clock clock = mo
[metron] branch master updated: METRON-2007 Management UI not loading grok statements correctly (merrimanr) closes apache/metron#1340
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 36946ff METRON-2007 Management UI not loading grok statements
correctly (merrimanr) closes apache/metron#1340
36946ff is described below
commit 36946ff09aadd54bebb503e89121d996f9242cc9
Author: merrimanr
AuthorDate: Tue Feb 19 10:49:00 2019 -0600
METRON-2007 Management UI not loading grok statements correctly (merrimanr)
closes apache/metron#1340
---
.../sensor-parser-config.component.html| 12
.../sensor-parser-config.component.spec.ts | 32 +++---
.../sensor-parser-config.component.ts | 29 ++--
.../src/app/service/grok-validation.service.ts | 2 +-
.../metron-config/src/app/service/hdfs.service.ts | 4 +--
.../metron-config/src/app/service/kafka.service.ts | 2 +-
6 files changed, 29 insertions(+), 52 deletions(-)
diff --git
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html
index c9895c7..b4bdd92 100644
---
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html
+++
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html
@@ -73,7 +73,7 @@
GROK STATEMENT
-
+
@@ -85,7 +85,7 @@
SCHEMA
-
+
TRANSFORMATIONS
{{getTransformationCount()}}
@@ -94,7 +94,7 @@
-
+
@@ -104,14 +104,14 @@
THREAT TRIAGE
-
+
RULES {{getRuleCount()}}
-
+
@@ -197,7 +197,7 @@
-SAVE
+SAVE
CANCEL
Advanced
diff --git
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts
index 4be0703..ff1a812 100644
---
a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts
+++
b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts
@@ -713,11 +713,9 @@ describe('Component: SensorParserConfig', () => {
it('should handle onSetKafkaTopic', async(() => {
spyOn(component, 'getKafkaStatus');
-spyOn(component, 'isConfigValid');
component.onSetKafkaTopic();
expect(component.getKafkaStatus).not.toHaveBeenCalled();
-expect(component.isConfigValid).toHaveBeenCalled();
component.sensorParserConfig.sensorTopic = 'bro';
component.onSetKafkaTopic();
@@ -726,10 +724,8 @@ describe('Component: SensorParserConfig', () => {
}));
it('should handle onSetSensorName', async(() => {
-spyOn(component, 'isConfigValid');
component.onSetSensorName();
-expect(component.isConfigValid).toHaveBeenCalled();
expect(component.sensorNameValid).toEqual(false);
component.sensorName = 'squid';
@@ -785,11 +781,9 @@ describe('Component: SensorParserConfig', () => {
it('should handle onParserTypeChange', async(() => {
spyOn(component, 'hidePane');
-spyOn(component, 'isConfigValid');
component.onParserTypeChange();
expect(component.hidePane).not.toHaveBeenCalled();
-expect(component.isConfigValid).toHaveBeenCalled();
component.sensorParserConfig.parserClassName =
'org.apache.metron.parsers.GrokParser';
@@ -803,37 +797,31 @@ describe('Component: SensorParserConfig', () => {
expect(component.hidePane).toHaveBeenCalledWith(Pane.GROK);
}));
- it('should handle onGrokStatementChange', async(() =>
[metron] branch master updated: METRON-1993 Stellar REST_GET should handle responses when content length is less than zero (merrimanr) closes apache/metron#1331
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 4d420f4 METRON-1993 Stellar REST_GET should handle responses when
content length is less than zero (merrimanr) closes apache/metron#1331
4d420f4 is described below
commit 4d420f4d1704b2da4e725de3fda4c08d97691611
Author: merrimanr
AuthorDate: Thu Feb 14 10:42:50 2019 -0600
METRON-1993 Stellar REST_GET should handle responses when content length is
less than zero (merrimanr) closes apache/metron#1331
---
.../metron/stellar/dsl/functions/RestConfig.java | 9 +++
.../stellar/dsl/functions/RestFunctions.java | 30 ++--
.../stellar/dsl/functions/RestFunctionsTest.java | 85 --
3 files changed, 111 insertions(+), 13 deletions(-)
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java
index fdb6935..610717e 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java
@@ -98,10 +98,15 @@ public class RestConfig extends HashMap {
*/
public final static String POOLING_DEFAULT_MAX_PER_RUOTE =
"pooling.default.max.per.route";
+ /**
+ * Setting this to true will verify the actual body content length equals
the content length header
+ */
+ public final static String VERIFY_CONTENT_LENGTH = "verify.content.length";
public RestConfig() {
put(TIMEOUT, 1000);
put(RESPONSE_CODES_ALLOWED, Collections.singletonList(200));
+put(VERIFY_CONTENT_LENGTH, false);
}
public String getBasicAuthUser() {
@@ -164,4 +169,8 @@ public class RestConfig extends HashMap {
public Integer getPoolingDefaultMaxPerRoute() {
return (Integer) get(POOLING_DEFAULT_MAX_PER_RUOTE);
}
+
+ public Boolean verifyContentLength() {
+return (Boolean) get(VERIFY_CONTENT_LENGTH);
+ }
}
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
index f07d54e..d6b03ce 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
@@ -244,15 +244,13 @@ public class RestFunctions {
scheduledFuture.cancel(true);
}
int statusCode = response.getStatusLine().getStatusCode();
+ LOG.debug("request = {}; response = {}", httpGet, response);
if (restConfig.getResponseCodesAllowed().contains(statusCode)) {
HttpEntity httpEntity = response.getEntity();
-// Parse the reponse if present, return the empty value override if not
-if (httpEntity != null && httpEntity.getContentLength() > 0) {
- String json = EntityUtils.toString(response.getEntity());
- return JSONUtils.INSTANCE.load(json, JSONUtils.MAP_SUPPLIER);
-}
-return restConfig.getEmptyContentOverride();
+// Parse the response if present, return the empty value override if
not
+Optional parsedResponse = parseResponse(restConfig, httpGet,
httpEntity);
+return parsedResponse.orElseGet(restConfig::getEmptyContentOverride);
} else {
throw new IOException(String.format("Stellar REST request to %s
expected status code to be one of %s but " +
"failed with http status code %d: %s",
@@ -374,6 +372,26 @@ public class RestFunctions {
return httpClientContext;
}
+protected Optional parseResponse(RestConfig restConfig, HttpGet
httpGet, HttpEntity httpEntity) throws IOException {
+ Optional parsedResponse = Optional.empty();
+ if (httpEntity != null) {
+int actualContentLength = 0;
+String json = EntityUtils.toString(httpEntity);
+if (json != null && !json.isEmpty()) {
+ actualContentLength = json.length();
+ parsedResponse = Optional.of(JSONUtils.INSTANCE.load(json,
JSONUtils.MAP_SUPPLIER));
+}
+if (restConfig.verifyContentLength() && actualContentLength !=
httpEntity.getContentLength()) {
+ throw new IOException(String.format("Stellar REST request to %s
returned incorrect or missing content length. " +
+ "Content length in the response was %d but the
actual body content length was %d.",
+ httpGet.getURI().toString(),
+ httpEntit
[metron] branch master updated: METRON-1948 Dropped messages from REGEX_SELECT parser field transformation are not acked in Storm (merrimanr) closes apache/metron#1321
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 01d6688 METRON-1948 Dropped messages from REGEX_SELECT parser field
transformation are not acked in Storm (merrimanr) closes apache/metron#1321
01d6688 is described below
commit 01d66880d99f5f46770b0f02755f4e9d86980cf5
Author: merrimanr
AuthorDate: Mon Jan 28 08:38:33 2019 -0600
METRON-1948 Dropped messages from REGEX_SELECT parser field transformation
are not acked in Storm (merrimanr) closes apache/metron#1321
---
.../integration/validation/StormParserDriver.java | 18 ---
.../apache/metron/writer/BulkWriterComponent.java | 16 --
.../metron/writer/BulkWriterComponentTest.java | 36 +-
3 files changed, 63 insertions(+), 7 deletions(-)
diff --git
a/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
index dfc454f..bfa1467 100644
---
a/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
+++
b/metron-platform/metron-parsing/metron-parsing-storm/src/test/java/org/apache/metron/parsers/integration/validation/StormParserDriver.java
@@ -25,14 +25,20 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
+
import org.apache.commons.lang.SerializationUtils;
+import org.apache.metron.common.configuration.IndexingConfigurations;
import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
+import org.apache.metron.common.writer.BulkMessageWriter;
+import org.apache.metron.common.writer.BulkWriterResponse;
import org.apache.metron.common.writer.MessageWriter;
import org.apache.metron.integration.ProcessorResult;
import org.apache.metron.parsers.bolt.ParserBolt;
import org.apache.metron.parsers.bolt.WriterHandler;
import org.apache.storm.task.OutputCollector;
+import org.apache.storm.task.TopologyContext;
import org.apache.storm.tuple.Tuple;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
@@ -41,7 +47,7 @@ import org.slf4j.LoggerFactory;
public class StormParserDriver extends ParserDriver {
private static final Logger LOG =
LoggerFactory.getLogger(StormParserDriver.class);
- public static class CollectingWriter implements MessageWriter {
+ public static class CollectingWriter implements
BulkMessageWriter {
List output;
public CollectingWriter(List output) {
@@ -49,13 +55,16 @@ public class StormParserDriver extends ParserDriver {
}
@Override
-public void init() {
+public void init(Map stormConf, TopologyContext topologyContext,
WriterConfiguration config) throws Exception {
}
@Override
-public void write(String sensorType, WriterConfiguration configurations,
Tuple tuple, JSONObject message) throws Exception {
- output.add(message.toJSONString().getBytes());
+public BulkWriterResponse write(String sensorType, WriterConfiguration
configurations, Iterable tuples, List messages) throws
Exception {
+ messages.forEach(message ->
output.add(message.toJSONString().getBytes()));
+ BulkWriterResponse bulkWriterResponse = new BulkWriterResponse();
+ bulkWriterResponse.addAllSuccesses(tuples);
+ return bulkWriterResponse;
}
@Override
@@ -83,6 +92,7 @@ public class StormParserDriver extends ParserDriver {
@Override
public ParserConfigurations getConfigurations() {
+
config.getSensorParserConfig(sensorType).getParserConfig().put(IndexingConfigurations.BATCH_SIZE_CONF,
1);
return config;
}
diff --git
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
index 68585c5..15e59d3 100644
---
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
+++
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
@@ -29,6 +29,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.metron.common.Constants;
import org.apache.metron.common.configuration.writer.WriterConfiguration;
@@ -126,8 +127,10 @@ public class BulkWriterComponent {
.withThrowable(e)
.addRawMessage(messageGetStrategy.get(t));
collector.e
[metron] branch master updated: METRON-1945 Metron MPack support for Knox SSO setup (merrimanr) closes apache/metron#1308
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 309ce65 METRON-1945 Metron MPack support for Knox SSO setup
(merrimanr) closes apache/metron#1308
309ce65 is described below
commit 309ce65f863d2532ca29a1c779bda02c1d42ee4d
Author: merrimanr
AuthorDate: Thu Jan 3 17:05:25 2019 -0600
METRON-1945 Metron MPack support for Knox SSO setup (merrimanr) closes
apache/metron#1308
---
.../CURRENT/configuration/metron-security-env.xml | 43 +
.../common-services/METRON/CURRENT/metainfo.xml| 3 +
.../CURRENT/package/scripts/alerts_ui_master.py| 10 +++
.../package/scripts/management_ui_master.py| 10 +++
.../CURRENT/package/scripts/metron_client.py | 21 +
.../CURRENT/package/scripts/metron_service.py | 39
.../CURRENT/package/scripts/params/params_linux.py | 47 +++---
.../package/scripts/params/status_params.py| 6 +-
.../METRON/CURRENT/package/scripts/rest_master.py | 4 +
.../package/templates/alerts-ui-app-config.json.j2 | 4 +
.../templates/management-ui-app-config.json.j2 | 4 +
.../METRON/CURRENT/package/templates/metron.xml.j2 | 56
.../CURRENT/package/templates/metronsso.xml.j2 | 100 +
.../METRON/CURRENT/themes/metron_theme.json| 51 ++-
metron-interface/README.md | 78
.../src/main/scripts/install_metron_knox.sh| 32 ---
16 files changed, 485 insertions(+), 23 deletions(-)
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml
index ab1fe6c..ab3e532 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-security-env.xml
@@ -182,5 +182,48 @@
+
+metron.knox.enabled
+Knox Enabled
+false
+Enable Knox
+
+ value-list
+
+
+ true
+ On
+
+
+ false
+ Off
+
+
+ 1
+ false
+
+
+
+
+metron.knox.sso.pubkey
+Knox SSO Public Key
+
+Knox public key used to verify Knox SSO tokens.
+
+
+ false
+ true
+
+
+
+metron.knox.sso.token.ttl
+Knox SSO Token Time to live
+30
+Controls the time to live for Knox SSO tokens in Metron.
Units are in milliseconds.
+
+
+ false
+
+
\ No newline at end of file
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 9516014..99e9325 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -416,6 +416,9 @@
PYTHON
1200
+
+metron-security-env
+
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/alerts_ui_master.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/alerts_ui_master.py
index 85c3be3..7cc2c31 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/alerts_ui_master.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/alerts_ui_master.py
@@ -19,6 +19,7 @@ limitations under the License.
from resource_management.core.exceptions import ComponentIsNotRunning
from resource_management.core.exceptions import ExecutionFailed
+from resource_management.core.exceptions import Fail
from resource_management.core.resources.system import Directory
from resource_management.core.resources.system import File
from resource_management.core.source import Template
@@ -51,6 +52,15 @@ class AlertsUIMaster(Script):
group=params.metron_group
)
+File(format("{metron_alerts_ui_path}/assets/app-config.json"),
+ content=Template("alerts-ui-app-config.json.j2"),
+ owner=params.metron_user,
+ group
[metron] branch master updated: METRON-1958 Optimize Cypress to use best practices (sardell via merrimanr) closes apache/metron#1317
This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 74658e9 METRON-1958 Optimize Cypress to use best practices (sardell
via merrimanr) closes apache/metron#1317
74658e9 is described below
commit 74658e97a55aeac612a70689850d352d907adf96
Author: sardell
AuthorDate: Thu Jan 3 09:40:57 2019 -0600
METRON-1958 Optimize Cypress to use best practices (sardell via merrimanr)
closes apache/metron#1317
---
.../CURRENT/package/templates/alerts_ui.yml.j2 | 2 ++
metron-interface/metron-alerts/alerts_ui.yml | 2 ++
.../{alerts_ui.yml => alerts_ui_cypress.yml} | 6 ++--
metron-interface/metron-alerts/cypress.json| 3 +-
.../cypress/integration/pcap/pcap.spec.js | 38 --
metron-interface/metron-alerts/package-lock.json | 10 ++
metron-interface/metron-alerts/package.json| 6 ++--
.../metron-alerts/scripts/alerts-server.js | 6 ++--
8 files changed, 48 insertions(+), 25 deletions(-)
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/alerts_ui.yml.j2
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/alerts_ui.yml.j2
index a3e87cb..92a9510 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/alerts_ui.yml.j2
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/alerts_ui.yml.j2
@@ -21,3 +21,5 @@ port: {{metron_alerts_ui_port}}
rest:
host: {{metron_rest_host}}
port: {{metron_rest_port}}
+
+dirPath: alerts-ui
diff --git a/metron-interface/metron-alerts/alerts_ui.yml
b/metron-interface/metron-alerts/alerts_ui.yml
index 4481fa9..343d883 100644
--- a/metron-interface/metron-alerts/alerts_ui.yml
+++ b/metron-interface/metron-alerts/alerts_ui.yml
@@ -19,3 +19,5 @@ port: 4201
rest:
host: localhost
port: 8080
+
+dirPath: alerts-ui
\ No newline at end of file
diff --git a/metron-interface/metron-alerts/alerts_ui.yml
b/metron-interface/metron-alerts/alerts_ui_cypress.yml
similarity index 95%
copy from metron-interface/metron-alerts/alerts_ui.yml
copy to metron-interface/metron-alerts/alerts_ui_cypress.yml
index 4481fa9..ddf663a 100644
--- a/metron-interface/metron-alerts/alerts_ui.yml
+++ b/metron-interface/metron-alerts/alerts_ui_cypress.yml
@@ -14,8 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-port: 4201
+port: 4200
rest:
host: localhost
- port: 8080
+ port: 4200
+
+dirPath: dist
\ No newline at end of file
diff --git a/metron-interface/metron-alerts/cypress.json
b/metron-interface/metron-alerts/cypress.json
index 7c0410d..da8aafb 100644
--- a/metron-interface/metron-alerts/cypress.json
+++ b/metron-interface/metron-alerts/cypress.json
@@ -3,5 +3,6 @@
"viewportHeight": 850,
"video": false,
"supportFile": false,
- "pluginsFile": false
+ "pluginsFile": false,
+ "baseUrl": "http://localhost:4200;
}
\ No newline at end of file
diff --git
a/metron-interface/metron-alerts/cypress/integration/pcap/pcap.spec.js
b/metron-interface/metron-alerts/cypress/integration/pcap/pcap.spec.js
index 58f7d26..835c714 100644
--- a/metron-interface/metron-alerts/cypress/integration/pcap/pcap.spec.js
+++ b/metron-interface/metron-alerts/cypress/integration/pcap/pcap.spec.js
@@ -25,8 +25,13 @@ context('PCAP Tab', () => {
url: '/api/v1/user',
response: 'user'
});
+cy.route({
+ method: 'POST',
+ url: 'logout',
+ response: []
+});
-cy.route('GET', 'config', 'fixture:config.json');
+cy.route('GET', '/api/v1/global/config', 'fixture:config.json');
cy.route('POST', 'search', 'fixture:search.json');
cy.route({
@@ -34,15 +39,14 @@ context('PCAP Tab', () => {
url: '/api/v1/pcap?state=*',
response: []
}).as('runningJobs');
-
-cy.visit('http://localhost:4200/login');
+
+cy.visit('login');
cy.get('[name="user"]').type('user');
cy.get('[name="password"]').type('password');
cy.contains('LOG IN').click();
});
afterEach(() => {
-cy.get('.logout-link').click();
});
it('checking running jobs on navigating to PCAP tab', () => {
@@ -62,7 +66,7 @@ context('PCAP Tab', () => {
cy.get('[data-qe-id="protocol"]').type('24');
cy.get('[data-qe-id="include-reverse"]').check();
cy.get('[data-qe-id="packet-filter"]').type('filter');
-
+
cy.get('[data-qe-id="submit-button"]').click();
cy.wait('@post
metron git commit: METRON-1850 Stellar REST function (merrimanr) closes apache/metron#1250
Repository: metron
Updated Branches:
refs/heads/master b5712af5b -> 3e73391ed
METRON-1850 Stellar REST function (merrimanr) closes apache/metron#1250
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3e73391e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3e73391e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3e73391e
Branch: refs/heads/master
Commit: 3e73391ed518e6d4e2f87a37745a34581782236b
Parents: b5712af
Author: merrimanr
Authored: Thu Nov 8 16:44:32 2018 -0600
Committer: rmerriman
Committed: Thu Nov 8 16:44:32 2018 -0600
--
metron-analytics/metron-maas-common/pom.xml | 2 +-
.../common/bolt/ConfiguredEnrichmentBolt.java | 14 +
.../common/bolt/ConfiguredParserBolt.java | 13 +
metron-platform/metron-data-management/pom.xml | 5 +-
.../enrichment/bolt/GenericEnrichmentBolt.java | 1 +
.../enrichment/bolt/ThreatIntelJoinBolt.java| 6 -
metron-stellar/stellar-common/README.md | 98 ++-
metron-stellar/stellar-common/pom.xml | 40 ++
.../stellar/common/shell/cli/StellarShell.java | 2 +
.../org/apache/metron/stellar/dsl/Context.java | 5 +
.../stellar/dsl/functions/RestConfig.java | 166 +
.../stellar/dsl/functions/RestFunctions.java| 388
.../dsl/functions/RestFunctionsTest.java| 604 +++
pom.xml | 1 +
14 files changed, 1334 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/3e73391e/metron-analytics/metron-maas-common/pom.xml
--
diff --git a/metron-analytics/metron-maas-common/pom.xml
b/metron-analytics/metron-maas-common/pom.xml
index b5cf1c9..062e3db 100644
--- a/metron-analytics/metron-maas-common/pom.xml
+++ b/metron-analytics/metron-maas-common/pom.xml
@@ -62,7 +62,7 @@
org.apache.httpcomponents
httpclient
- 4.3.2
+ ${global_httpclient_version}
org.codehaus.jackson
http://git-wip-us.apache.org/repos/asf/metron/blob/3e73391e/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java
--
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java
index c28ca7b..2e03a36 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java
@@ -17,8 +17,10 @@
*/
package org.apache.metron.common.bolt;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import org.apache.metron.common.configuration.EnrichmentConfigurations;
+import org.apache.metron.stellar.dsl.StellarFunctions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,4 +33,16 @@ public abstract class ConfiguredEnrichmentBolt extends
ConfiguredBolthttps://storm.apache.org/releases/1.0.6/javadocs/org/apache/storm/task/IBolt.html#cleanup--
for more detail.
+super.cleanup();
+try {
+ StellarFunctions.close();
+} catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+}
+ }
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3e73391e/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java
--
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java
index 14ce50b..17b614b 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java
@@ -17,9 +17,11 @@
*/
package org.apache.metron.common.bolt;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import org.apache.metron.common.configuration.ParserConfigurations;
import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.stellar.dsl.StellarFunctions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,4 +38,15 @@ public abstract class ConfiguredParserBolt extends
ConfiguredBolthttps://storm.apache.org/releases/1.0.6/javadocs/org/apache/storm/task/IBolt.html#cleanup--
for more detail.
+super.cleanup();
+try {
+ StellarFunctions.close();
+} ca
metron git commit: METRON-1830 Re-implement Alerts dialog box without jQuery (sardell via merrimanr) closes apache/metron#1240
Repository: metron
Updated Branches:
refs/heads/master 0e037edad -> eee996302
METRON-1830 Re-implement Alerts dialog box without jQuery (sardell via
merrimanr) closes apache/metron#1240
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/eee99630
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/eee99630
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/eee99630
Branch: refs/heads/master
Commit: eee99630270e08412e589b99afafbb2246f88af5
Parents: 0e037ed
Author: sardell
Authored: Thu Oct 25 07:59:44 2018 -0500
Committer: rmerriman
Committed: Thu Oct 25 07:59:44 2018 -0500
--
.../alert-details/alert-details.component.ts| 10 +-
.../alerts/alerts-list/alerts-list.component.ts | 9 +-
.../table-view/table-view.component.spec.ts | 4 +-
.../table-view/table-view.component.ts | 49 ---
.../tree-view/tree-view.component.spec.ts | 4 +-
.../tree-view/tree-view.component.ts| 20 +--
.../alerts/meta-alerts/meta-alerts.component.ts | 2 -
.../save-search/save-search.component.spec.ts | 4 +-
.../alerts/save-search/save-search.component.ts | 12 +-
.../saved-searches.component.spec.ts| 5 +-
.../saved-searches/saved-searches.component.ts | 41 --
.../metron-alerts/src/app/app.component.html| 1 +
.../metron-alerts/src/app/app.component.spec.ts | 4 +
.../metron-alerts/src/app/app.module.ts | 12 +-
.../src/app/model/confirmation-type.ts | 21 +++
.../metron-alerts/src/app/model/dialog-type.ts | 21 +++
.../src/app/service/dialog.service.spec.ts | 78
.../src/app/service/dialog.service.ts | 55
.../src/app/shared/metron-dialog-box.ts | 91 -
.../metron-dialog/metron-dialog.component.html | 40 ++
.../metron-dialog/metron-dialog.component.scss | 20 +++
.../metron-dialog.component.spec.ts | 127 +++
.../metron-dialog/metron-dialog.component.ts| 66 ++
23 files changed, 529 insertions(+), 167 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/eee99630/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
index 0693457..4c0b18b 100644
---
a/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
+++
b/metron-interface/metron-alerts/src/app/alerts/alert-details/alert-details.component.ts
@@ -29,10 +29,11 @@ import {PatchRequest} from '../../model/patch-request';
import {Patch} from '../../model/patch';
import {AlertComment} from './alert-comment';
import {AuthenticationService} from '../../service/authentication.service';
-import {MetronDialogBox} from '../../shared/metron-dialog-box';
import {CommentAddRemoveRequest} from "../../model/comment-add-remove-request";
import {META_ALERTS_SENSOR_TYPE} from '../../utils/constants';
import {GlobalConfigService} from '../../service/global-config.service';
+import { DialogService } from 'app/service/dialog.service';
+import { ConfirmationType } from 'app/model/confirmation-type';
export enum AlertState {
NEW, OPEN, ESCALATE, DISMISS, RESOLVE
@@ -83,7 +84,7 @@ export class AlertDetailsComponent implements OnInit {
private updateService: UpdateService,
private alertsService: AlertsService,
private authenticationService: AuthenticationService,
- private metronDialogBox: MetronDialogBox,
+ private dialogService: DialogService,
globalConfigService: GlobalConfigService) {
this.globalConfigService = globalConfigService;
}
@@ -249,8 +250,8 @@ export class AlertDetailsComponent implements OnInit {
commentText += ' \'' +
this.alertCommentsWrapper[index].alertComment.comment + '\'';
}
-
this.metronDialogBox.showConfirmationMessage(commentText).subscribe(response =>
{
- if (response) {
+const confirmedSubscription =
this.dialogService.launchDialog(commentText).subscribe(action => {
+ if (action === ConfirmationType.Confirmed) {
let deletedCommentWrapper = this.alertCommentsWrapper.splice(index,
1)[0];
let commentRequest = new CommentAddRemoveRequest();
commentRequest.guid = this.alertSource.guid;
@@ -268,6 +269,7 @@ export class AlertDetailsComponent implements OnInit {
this.alertCommentsWrapper.sort((a, b) =>
b.alertComment.timestamp - a.alertComment.timestamp);
});
}
+ confirmedSu
metron git commit: METRON-1829 Large Error Message Causes Slow Search Performance (merrimanr) closes apache/metron#1239
Repository: metron
Updated Branches:
refs/heads/master 2531c3e19 -> d44a39256
METRON-1829 Large Error Message Causes Slow Search Performance (merrimanr)
closes apache/metron#1239
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d44a3925
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d44a3925
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d44a3925
Branch: refs/heads/master
Commit: d44a392567e35022bfb35a787b55aff3447ac60e
Parents: 2531c3e
Author: merrimanr
Authored: Mon Oct 22 08:43:10 2018 -0500
Committer: rmerriman
Committed: Mon Oct 22 08:43:10 2018 -0500
--
.../bolt/BulkMessageWriterBoltTest.java | 2 +-
metron-platform/metron-writer/pom.xml | 6 ++
.../metron/writer/BulkWriterComponent.java | 33 ++-
.../writer/bolt/BulkMessageWriterBolt.java | 2 +-
.../metron/writer/BulkWriterComponentTest.java | 61 +++-
5 files changed, 76 insertions(+), 28 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
--
diff --git
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
index 588fc58..083628c 100644
---
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
+++
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/bolt/BulkMessageWriterBoltTest.java
@@ -207,7 +207,7 @@ public class BulkMessageWriterBoltTest extends
BaseEnrichmentBoltTest {
}
UnitTestHelper.setLog4jLevel(BulkWriterComponent.class, Level.ERROR);
verify(outputCollector, times(5)).ack(tuple);
-verify(outputCollector, times(1)).emit(eq(Constants.ERROR_STREAM),
any(Values.class));
+verify(outputCollector, times(5)).emit(eq(Constants.ERROR_STREAM),
any(Values.class));
verify(outputCollector, times(1)).reportError(any(Throwable.class));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-writer/pom.xml
--
diff --git a/metron-platform/metron-writer/pom.xml
b/metron-platform/metron-writer/pom.xml
index e845516..6d08093 100644
--- a/metron-platform/metron-writer/pom.xml
+++ b/metron-platform/metron-writer/pom.xml
@@ -207,6 +207,12 @@
metron-common
${project.parent.version}
+
+org.apache.metron
+metron-test-utilities
+${project.parent.version}
+test
+
http://git-wip-us.apache.org/repos/asf/metron/blob/d44a3925/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
--
diff --git
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
index 7678584..68585c5 100644
---
a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
+++
b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/BulkWriterComponent.java
@@ -40,6 +40,7 @@ import org.apache.metron.common.writer.BulkMessageWriter;
import org.apache.metron.common.writer.BulkWriterResponse;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.tuple.Tuple;
+import org.apache.storm.tuple.Values;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -118,12 +119,18 @@ public class BulkWriterComponent {
public void error(String sensorType, Throwable e, Iterable tuples,
MessageGetStrategy messageGetStrategy) {
LOG.error(format("Failing %d tuple(s); sensorType=%s",
Iterables.size(tuples), sensorType), e);
-MetronError error = new MetronError()
-.withSensorType(Collections.singleton(sensorType))
-.withErrorType(Constants.ErrorType.INDEXING_ERROR)
-.withThrowable(e);
-tuples.forEach(t -> error.addRawMessage(messageGetStrategy.get(t)));
-handleError(tuples, error);
+tuples.forEach(t -> {
+ MetronError error = new MetronError()
+ .withSensorType(Collections.singleton(sensorType))
+ .withErrorType(Constants.ErrorType.INDEXING_ERROR)
+ .withThrowable(e)
+ .addRawMessage(messageGetStrategy.get(t));
+ collector.emit(Constants.ERROR_
metron git commit: METRON-1816 Date format Stellar function (merrimanr) closes apache/metron#1233
Repository: metron
Updated Branches:
refs/heads/master 28542ad64 -> bf32fef6a
METRON-1816 Date format Stellar function (merrimanr) closes apache/metron#1233
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bf32fef6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bf32fef6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bf32fef6
Branch: refs/heads/master
Commit: bf32fef6a4a02c5b21ec1f6d97fbd83de78a996e
Parents: 28542ad
Author: merrimanr
Authored: Thu Oct 18 17:00:44 2018 -0500
Committer: rmerriman
Committed: Thu Oct 18 17:00:44 2018 -0500
--
metron-stellar/stellar-common/README.md | 9 +
.../stellar/dsl/functions/DateFunctions.java| 42
.../dsl/functions/DateFunctionsTest.java| 38 ++
3 files changed, 89 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/bf32fef6/metron-stellar/stellar-common/README.md
--
diff --git a/metron-stellar/stellar-common/README.md
b/metron-stellar/stellar-common/README.md
index 468f358..0f3bb6a 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -169,6 +169,7 @@ Where:
| [ `CHOP`](#chop)
|
| [ `CHOMP`](#chomp)
|
| [ `COUNT_MATCHES`](#count_matches)
|
+| [ `DATE_FORMAT`](#date_format)
| [ `DAY_OF_MONTH`](#day_of_month)
|
| [ `DAY_OF_WEEK`](#day_of_week)
|
| [ `DAY_OF_YEAR`](#day_of_year)
|
@@ -379,6 +380,14 @@ Where:
* substring/character - the substring or character to count, may be null.
* Returns: the number of non-overlapping occurrences, 0 if either
CharSequence is null.
+### `DATE_FORMAT`
+ * Description: Takes an epoch timestamp and converts it to a date format.
+ * Input:
+* format - DateTime format as a String.
+* timestampField - Optional epoch time in Long format. Defaults to now.
+* timezone - Optional timezone in String format.
+ * Returns: Formatted date.
+
### `DAY_OF_MONTH`
* Description: The numbered day within the month. The first day within the
month has a value of 1.
* Input:
http://git-wip-us.apache.org/repos/asf/metron/blob/bf32fef6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
--
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
index 212d6e9..17f5f8d 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DateFunctions.java
@@ -28,6 +28,7 @@ import org.apache.metron.stellar.common.utils.ConversionUtils;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
+import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.TimeZone;
@@ -109,6 +110,13 @@ public class DateFunctions {
return sdf.parse(date).getTime();
}
+ public static String getDateFormat(String format, Optional epochTime,
Optional timezone) {
+Long time = epochTime.orElseGet(System::currentTimeMillis);
+TimezonedFormat fmt = timezone.map(s -> new TimezonedFormat(format,
s)).orElseGet(() -> new TimezonedFormat(format));
+SimpleDateFormat sdf = formatCache.get(fmt).get();
+return sdf.format(new Date(time));
+ }
+
/**
* Stellar Function: TO_EPOCH_TIMESTAMP
@@ -144,6 +152,40 @@ public class DateFunctions {
}
}
+ @Stellar( name="DATE_FORMAT",
+ description = "Takes an epoch timestamp and converts it to a date
format.",
+ params = {"format - DateTime format as a String."
+ , "timestampField - Optional epoch time in Long format.
Defaults to now."
+ , "timezone - Optional timezone in String format."},
+ returns="Formatted date."
+ )
+ public static class DateFormat extends BaseStellarFunction {
+
+public Object apply(List objects) {
+ int size = objects.size();
+ Optional formatObj = Optional.
[2/2] metron git commit: METRON-1681 Decouple the ParserBolt from the Parse execution logic (merrimanr) closes apache/metron#1213
METRON-1681 Decouple the ParserBolt from the Parse execution logic (merrimanr)
closes apache/metron#1213
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/28542ad6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/28542ad6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/28542ad6
Branch: refs/heads/master
Commit: 28542ad64cf63f17b728b4b1c0e995a8973767f7
Parents: 08f3de0
Author: merrimanr
Authored: Thu Oct 18 13:59:52 2018 -0500
Committer: rmerriman
Committed: Thu Oct 18 13:59:52 2018 -0500
--
.../impl/SensorParserConfigServiceImpl.java | 51 +-
.../parsers/DefaultParserRunnerResults.java | 71 ++
.../org/apache/metron/parsers/GrokParser.java | 3 +-
.../org/apache/metron/parsers/ParserRunner.java | 60 ++
.../apache/metron/parsers/ParserRunnerImpl.java | 322 +++
.../metron/parsers/ParserRunnerResults.java | 33 +
.../apache/metron/parsers/bolt/ParserBolt.java | 381 +++-
.../parsers/filters/BroMessageFilter.java | 2 +-
.../metron/parsers/filters/StellarFilter.java | 2 +-
.../parsers/interfaces/MessageFilter.java | 2 +-
.../parsers/interfaces/MessageParser.java | 27 +-
.../interfaces/MultilineMessageParser.java | 51 --
.../metron/parsers/syslog/Syslog5424Parser.java | 4 +-
.../parsers/topology/ParserComponent.java | 56 ++
.../parsers/topology/ParserComponents.java | 67 --
.../parsers/topology/ParserTopologyBuilder.java | 39 +-
.../org/apache/metron/filters/FiltersTest.java | 4 +-
.../metron/parsers/MessageParserTest.java | 108 ++-
.../metron/parsers/ParserRunnerImplTest.java| 390 +
.../metron/parsers/bolt/ParserBoltTest.java | 859 ++-
.../parsers/integration/ParserDriver.java | 60 +-
21 files changed, 1481 insertions(+), deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
index 4cd272e..d0e4b3d 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java
@@ -20,12 +20,10 @@ package org.apache.metron.rest.service.impl;
import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME;
import com.fasterxml.jackson.databind.ObjectMapper;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Set;
import org.apache.curator.framework.CuratorFramework;
import org.apache.hadoop.fs.Path;
import org.apache.metron.common.configuration.ConfigurationType;
@@ -35,18 +33,14 @@ import
org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.common.zookeeper.ConfigurationsCache;
import org.apache.metron.parsers.interfaces.MessageParser;
import org.apache.metron.parsers.interfaces.MessageParserResult;
-import org.apache.metron.parsers.interfaces.MultilineMessageParser;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.ParseMessageRequest;
import org.apache.metron.rest.service.GrokService;
import org.apache.metron.rest.service.SensorParserConfigService;
import org.apache.metron.rest.util.ParserIndex;
-import org.apache.metron.common.zookeeper.ZKConfigurationsCache;
import org.apache.zookeeper.KeeperException;
import org.json.simple.JSONObject;
-import org.reflections.Reflections;
-import org.reflections.util.ConfigurationBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -141,53 +135,13 @@ public class SensorParserConfigServiceImpl implements
SensorParserConfigService
} else if (sensorParserConfig.getParserClassName() == null) {
throw new RestException("SensorParserConfig must have a
parserClassName");
} else {
- MultilineMessageParser parser;
- Object parserObject;
+ MessageParser parser;
try {
-parserObject = Class.forName(sensorParserConfig.getParserClassName())
+parser = (MessageParser)
Class.forName(sensorParserConfig.getParserClassName())
.newInstance();
} catch (Exception e) {
throw new RestException(e.toString(),
[1/2] metron git commit: METRON-1681 Decouple the ParserBolt from the Parse execution logic (merrimanr) closes apache/metron#1213
Repository: metron
Updated Branches:
refs/heads/master 08f3de0fe -> 28542ad64
http://git-wip-us.apache.org/repos/asf/metron/blob/28542ad6/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
--
diff --git
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
new file mode 100644
index 000..5f05b24
--- /dev/null
+++
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/ParserRunnerImplTest.java
@@ -0,0 +1,390 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.ParserConfigurations;
+import org.apache.metron.common.configuration.SensorParserConfig;
+import org.apache.metron.common.error.MetronError;
+import org.apache.metron.common.message.metadata.RawMessage;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.common.utils.ReflectionUtils;
+import org.apache.metron.parsers.filters.Filters;
+import org.apache.metron.parsers.filters.StellarFilter;
+import org.apache.metron.parsers.interfaces.MessageFilter;
+import org.apache.metron.parsers.interfaces.MessageParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.parsers.topology.ParserComponent;
+import org.apache.metron.parsers.ParserRunnerImpl.ProcessResult;
+import org.apache.metron.stellar.dsl.Context;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
+import static org.powermock.api.mockito.PowerMockito.when;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ParserRunnerImpl.class, ReflectionUtils.class, Filters.class})
+public class ParserRunnerImplTest {
+
+ @Rule
+ public final ExpectedException exception = ExpectedException.none();
+
+ /**
+ {
+ "fieldValidations" : [
+ {
+ "input" : [ "ip_src_addr", "ip_dst_addr"],
+ "validation" : "IP"
+ }
+ ]
+ }
+ */
+ @Multiline
+ private String globalConfigString;
+
+ /**
+ {
+ "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
+ "filterClassName":"org.apache.metron.parsers.filters.StellarFilter",
+ "sensorTopic":"bro",
+ "parserConfig": {
+ "field": "value"
+ },
+ "fieldTransformations" : [
+ {
+ "input" : "field1",
+ "transformation" : "REMOVE"
+ }
+ ]
+ }
+ */
+ @Multiline
+ private String broConfigString;
+
+ /**
+ {
+ "parserClassName":"org.apache.metron.parsers.snort.BasicSnortParser",
+ "sensorTopic":"snort",
+ "parserConfig": {}
+ }
+ */
+ @Multiline
+ private String snortConfigString;
+
+ private ParserConfigurations parserConfigurations;
+ private MessageParser broParser;
+ private MessageParser snortParser;
+ private MessageFilter stellarFilter;
+ private ParserRunnerImpl parserRunner;
+
+
+ @Before
+ public void setup() throws IOException {
+parserConfigurations = new ParserConfigurations();
+SensorParserConfig broConfig =
SensorParserConfig.fromBytes(broConfigString.getBytes());
+SensorParserConfig
metron git commit: METRON-1805 Provide a default value for the Storm topology.max.spout.pending setting (merrimanr) closes apache/metron#1221
Repository: metron Updated Branches: refs/heads/master 647896604 -> 621415067 METRON-1805 Provide a default value for the Storm topology.max.spout.pending setting (merrimanr) closes apache/metron#1221 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/62141506 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/62141506 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/62141506 Branch: refs/heads/master Commit: 621415067dc727838da8e14925c15120b0e479de Parents: 6478966 Author: merrimanr Authored: Mon Oct 15 14:50:10 2018 -0500 Committer: rmerriman Committed: Mon Oct 15 14:50:10 2018 -0500 -- .../METRON/CURRENT/configuration/metron-indexing-env.xml | 4 ++-- .../METRON/CURRENT/configuration/metron-profiler-env.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/62141506/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml index 3d268b7..ebc9759 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml @@ -146,7 +146,7 @@ ra_indexing_topology_max_spout_pending Indexing Topology Spout Max Pending Tuples - +300 Indexing Max Pending for Random Access true @@ -155,7 +155,7 @@ batch_indexing_topology_max_spout_pending Indexing Topology Spout Max Pending Tuples - +300 Indexing Max Pending for HDFS true http://git-wip-us.apache.org/repos/asf/metron/blob/62141506/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml index 07b8d11..c45d799 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-profiler-env.xml @@ -165,7 +165,7 @@ profiler_topology_max_spout_pending Profiler Topology Spout Max Pending Tuples Spout Max Pending Tuples - +300 true
metron git commit: METRON-1811 Alert Search Fails When Sorting by Alert Status (merrimanr) closes apache/metron#1231
Repository: metron
Updated Branches:
refs/heads/master 3467ffdb1 -> 747220f00
METRON-1811 Alert Search Fails When Sorting by Alert Status (merrimanr) closes
apache/metron#1231
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/747220f0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/747220f0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/747220f0
Branch: refs/heads/master
Commit: 747220f000188557bfa860b48f6eb4bc49758d4a
Parents: 3467ffd
Author: merrimanr
Authored: Tue Oct 9 15:33:39 2018 -0500
Committer: rmerriman
Committed: Tue Oct 9 15:33:39 2018 -0500
--
.../CURRENT/package/files/bro_index.template| 3 +
.../CURRENT/package/files/snort_index.template | 3 +
.../CURRENT/package/files/yaf_index.template| 3 +
metron-platform/metron-elasticsearch/README.md | 20 +-
.../ElasticsearchSearchIntegrationTest.java | 211 ++-
.../indexing/dao/SearchIntegrationTest.java | 34 +--
.../src/main/config/schema/bro/schema.xml | 1 +
.../src/main/config/schema/snort/schema.xml | 1 +
.../src/main/config/schema/yaf/schema.xml | 1 +
.../integration/SolrSearchIntegrationTest.java | 8 +-
10 files changed, 112 insertions(+), 173 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
index 17ad4d2..48e76c7 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/bro_index.template
@@ -133,6 +133,9 @@
"source:type": {
"type": "keyword"
},
+"alert_status": {
+ "type": "keyword"
+},
/*
* Widely-used Bro fields (potentially renamed during Metron ingest)
*/
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
index f7c6e59..139dbc9 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/snort_index.template
@@ -123,6 +123,9 @@
"source:type": {
"type": "keyword"
},
+"alert_status": {
+ "type": "keyword"
+},
"ip_dst_addr": {
"type": "ip"
},
http://git-wip-us.apache.org/repos/asf/metron/blob/747220f0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
index f4093ba..4a3e5f9 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/yaf_index.template
@@ -123,6 +123,9 @@
"source:type": {
"type": "keyword"
},
+"alert_status": {
+ "type": "keyword"
+},
"ip_dst_addr": {
"type": "ip"
},
http://git-wip-us.apache.org/repos/asf/metron/blob
metron git commit: METRON-1807 Auto populate the recommended values to some of the metron config parameters (MohanDV via merrimanr) closes apache/metron#1227
Repository: metron
Updated Branches:
refs/heads/master 891bf4952 -> 269b91d01
METRON-1807 Auto populate the recommended values to some of the metron config
parameters (MohanDV via merrimanr) closes apache/metron#1227
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/269b91d0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/269b91d0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/269b91d0
Branch: refs/heads/master
Commit: 269b91d015c67b0ae49a78e5c457408e1b431c25
Parents: 891bf49
Author: MohanDV
Authored: Mon Oct 8 11:20:53 2018 -0500
Committer: rmerriman
Committed: Mon Oct 8 11:20:53 2018 -0500
--
.../METRON/CURRENT/configuration/metron-env.xml | 12
.../common-services/METRON/CURRENT/service_advisor.py | 10 ++
2 files changed, 18 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/269b91d0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index cdef7cf..81dda6c 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -130,17 +130,21 @@
true
-
+
storm_rest_addr
Storm Rest Server Address
URL of Storm UI (http://storm.ui.hostname:8744). If no
protocol is provided, http is assumed.
-
+
+true
+
-
+
zeppelin_server_url
Zeppelin Server Address
URL of Zeppelin
(zeppelin.server.addr:zeppelin.server.port)
-
+
+true
+
metron_principal_name
http://git-wip-us.apache.org/repos/asf/metron/blob/269b91d0/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
index b008fc0..36cec7c 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
@@ -158,6 +158,16 @@ class
METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
putMetronEnvProperty = self.putProperty(configurations,
"metron-env", services)
putMetronEnvProperty("zeppelin_server_url", zeppelinServerUrl)
+#Suggest Zookeeper quorum
+if "solr-cloud" in services["configurations"]:
+zookeeperHost = self.getComponentHostNames(services, "ZOOKEEPER",
"ZOOKEEPER_SERVER")[0]
+zookeeperClientPort =
services["configurations"]["zoo.cfg"]["properties"]["clientPort"]
+solrZkDir =
services["configurations"]["solr-cloud"]["properties"]["solr_cloud_zk_directory"]
+solrZookeeperUrl = zookeeperHost + ":" + zookeeperClientPort +
solrZkDir
+putMetronEnvProperty = self.putProperty(configurations,
"metron-env", services)
+putMetronEnvProperty("solr_zookeeper_url", solrZookeeperUrl)
+
+
def validateSTORMSiteConfigurations(self, properties, recommendedDefaults,
configurations, services, hosts):
# Determine if the cluster is secured
is_secured = self.isSecurityEnabled(services)
[1/3] metron git commit: METRON-1771 Update REST endpoints to support eventually consistent UI updates (merrimanr) closes apache/metron#1190
Repository: metron
Updated Branches:
refs/heads/master 7e222fa47 -> de533063c
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
--
diff --git
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
index 2f83921..54b5b64 100644
---
a/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
+++
b/metron-platform/metron-solr/src/main/java/org/apache/metron/solr/dao/SolrUpdateDao.java
@@ -56,7 +56,7 @@ public class SolrUpdateDao implements UpdateDao {
}
@Override
- public void update(Document update, Optional rawIndex) throws
IOException {
+ public Document update(Document update, Optional rawIndex) throws
IOException {
Document newVersion = update;
// Handle any case where we're given comments in Map form, instead of raw
String
Object commentsObj = update.getDocument().get(COMMENTS_FIELD);
@@ -79,10 +79,11 @@ public class SolrUpdateDao implements UpdateDao {
} catch (SolrServerException e) {
throw new IOException(e);
}
+return newVersion;
}
@Override
- public void batchUpdate(Map> updates) throws
IOException {
+ public Map> batchUpdate(Map> updates) throws IOException {
// updates with a collection specified
Map> solrCollectionUpdates = new
HashMap<>();
Set collectionsUpdated = new HashSet<>();
@@ -117,18 +118,20 @@ public class SolrUpdateDao implements UpdateDao {
} catch (SolrServerException e) {
throw new IOException(e);
}
+return updates;
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request) throws
IOException {
+ public Document addCommentToAlert(CommentAddRemoveRequest request) throws
IOException {
Document latest = retrieveLatestDao.getLatest(request.getGuid(),
request.getSensorType());
-addCommentToAlert(request, latest);
+return addCommentToAlert(request, latest);
}
@Override
- public void addCommentToAlert(CommentAddRemoveRequest request, Document
latest) throws IOException {
-if (latest == null) {
- return;
+ public Document addCommentToAlert(CommentAddRemoveRequest request, Document
latest) throws IOException {
+if (latest == null || latest.getDocument() == null) {
+ throw new IOException(String.format("Unable to add comment. Document
with guid %s cannot be found.",
+ request.getGuid()));
}
@SuppressWarnings("unchecked")
@@ -149,21 +152,22 @@ public class SolrUpdateDao implements UpdateDao {
Document newVersion = new Document(latest);
newVersion.getDocument().put(COMMENTS_FIELD, commentStrs);
-update(newVersion, Optional.empty());
+return update(newVersion, Optional.empty());
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request)
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request)
throws IOException {
Document latest = retrieveLatestDao.getLatest(request.getGuid(),
request.getSensorType());
-removeCommentFromAlert(request, latest);
+return removeCommentFromAlert(request, latest);
}
@Override
- public void removeCommentFromAlert(CommentAddRemoveRequest request, Document
latest)
+ public Document removeCommentFromAlert(CommentAddRemoveRequest request,
Document latest)
throws IOException {
-if (latest == null) {
- return;
+if (latest == null || latest.getDocument() == null) {
+ throw new IOException(String.format("Unable to remove comment. Document
with guid %s cannot be found.",
+ request.getGuid()));
}
@SuppressWarnings("unchecked")
@@ -171,8 +175,8 @@ public class SolrUpdateDao implements UpdateDao {
.get(COMMENTS_FIELD);
// Can't remove anything if there's nothing there
if (commentMap == null) {
- LOG.debug("Provided alert had no comments to be able to remove from");
- return;
+ throw new IOException(String.format("Unable to remove comment. Document
with guid %s has no comments.",
+ request.getGuid()));
}
List> originalComments = new ArrayList<>(commentMap);
List comments = new ArrayList<>();
@@ -186,7 +190,7 @@ public class SolrUpdateDao implements UpdateDao {
.collect(Collectors.toList());
Document newVersion = new Document(latest);
newVersion.getDocument().put(COMMENTS_FIELD, commentsAsJson);
-update(newVersion, Optional.empty());
+return update(newVersion, Optional.empty());
}
public void convertCommentsToRaw(Map source) {
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-solr/src/test/java/org/apache/metron/solr/dao/SolrMetaAlertDaoTest.java
[2/3] metron git commit: METRON-1771 Update REST endpoints to support eventually consistent UI updates (merrimanr) closes apache/metron#1190
http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
--
diff --git
a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
index 4d48075..096baf1 100644
---
a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
+++
b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/metaalert/lucene/AbstractLuceneMetaAlertUpdateDao.java
@@ -24,16 +24,17 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
import org.apache.metron.common.Constants;
-import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.indexing.dao.RetrieveLatestDao;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConfig;
import org.apache.metron.indexing.dao.metaalert.MetaAlertConstants;
@@ -80,15 +81,16 @@ public abstract class AbstractLuceneMetaAlertUpdateDao
implements MetaAlertUpdat
* @param retrieveLatestDao DAO to retrieve the item to be patched
* @param request The patch request.
* @param timestamp Optionally a timestamp to set. If not specified then
current time is used.
+ * @return The patched document
* @throws OriginalNotFoundException If no original document is found to
patch.
* @throws IOException If an error occurs performing the patch.
*/
@Override
- public void patch(RetrieveLatestDao retrieveLatestDao, PatchRequest request,
+ public Document patch(RetrieveLatestDao retrieveLatestDao, PatchRequest
request,
Optional timestamp)
throws OriginalNotFoundException, IOException {
if (isPatchAllowed(request)) {
- updateDao.patch(retrieveLatestDao, request, timestamp);
+ return updateDao.patch(retrieveLatestDao, request, timestamp);
} else {
throw new IllegalArgumentException(
"Meta alert patches are not allowed for /alert or /status paths. "
@@ -97,7 +99,7 @@ public abstract class AbstractLuceneMetaAlertUpdateDao
implements MetaAlertUpdat
}
@Override
- public void batchUpdate(Map> updates) {
+ public Map> batchUpdate(Map> updates) {
throw new UnsupportedOperationException("Meta alerts do not allow for bulk
updates");
}
@@ -170,21 +172,65 @@ public abstract class AbstractLuceneMetaAlertUpdateDao
implements MetaAlertUpdat
return updates;
}
+ /**
+ * Adds alerts to a metaalert, based on a list of GetRequests provided for
retrieval.
+ * @param metaAlertGuid The GUID of the metaalert to be given new children.
+ * @param alertRequests GetRequests for the appropriate alerts to add.
+ * @return The updated metaalert with alerts added.
+ */
+ @Override
+ public Document addAlertsToMetaAlert(String metaAlertGuid, List
alertRequests)
+ throws IOException {
+Document metaAlert = retrieveLatestDao
+.getLatest(metaAlertGuid, MetaAlertConstants.METAALERT_TYPE);
+if (metaAlert == null) {
+ throw new IOException(String.format("Unable to add alerts to meta alert.
Meta alert with guid %s cannot be found.",
+ metaAlertGuid));
+}
+if (MetaAlertStatus.ACTIVE.getStatusString()
+
.equals(metaAlert.getDocument().get(MetaAlertConstants.STATUS_FIELD))) {
+ Iterable alerts =
retrieveLatestDao.getAllLatest(alertRequests);
+ Set missingAlerts = getMissingAlerts(alertRequests, alerts);
+ if (!missingAlerts.isEmpty()) {
+throw new IOException(String.format("Unable to add alerts to meta
alert. Alert with guid %s cannot be found.",
+missingAlerts.iterator().next()));
+ }
+ Map> updates =
buildAddAlertToMetaAlertUpdates(metaAlert, alerts);
+ update(updates);
+ return metaAlert;
+} else {
+ throw new IllegalStateException("Adding alerts to an INACTIVE meta alert
is not allowed");
+}
+ }
+
+ /**
+ * Removes alerts from a metaalert, based on a list of GetRequests provided
for retrieval.
+ * @param metaAlertGuid The GUID of the metaalert to remove children from.
+ * @param alertRequests A list of GetReqests that will provide the alerts to
remove
+ * @return The updated metaalert with alerts removed.
+ * @throws IllegalStateException If the metaalert is inactive.
+ */
[3/3] metron git commit: METRON-1771 Update REST endpoints to support eventually consistent UI updates (merrimanr) closes apache/metron#1190
METRON-1771 Update REST endpoints to support eventually consistent UI updates (merrimanr) closes apache/metron#1190 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/de533063 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/de533063 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/de533063 Branch: refs/heads/master Commit: de533063c26c8a2462dbfff33228a949c4a96c97 Parents: 7e222fa Author: merrimanr Authored: Thu Oct 4 17:16:18 2018 -0500 Committer: rmerriman Committed: Thu Oct 4 17:16:18 2018 -0500 -- metron-interface/metron-rest/README.md | 16 ++ .../rest/controller/MetaAlertController.java| 18 +- .../rest/controller/UpdateController.java | 31 ++- .../metron/rest/service/MetaAlertService.java | 12 +- .../metron/rest/service/UpdateService.java | 9 +- .../rest/service/impl/MetaAlertServiceImpl.java | 18 +- .../rest/service/impl/UpdateServiceImpl.java| 18 +- .../MetaAlertControllerIntegrationTest.java | 152 +++--- .../elasticsearch/dao/ElasticsearchDao.java | 32 +-- .../dao/ElasticsearchMetaAlertDao.java | 37 ++-- .../dao/ElasticsearchMetaAlertUpdateDao.java| 55 ++--- .../dao/ElasticsearchUpdateDao.java | 42 ++-- .../dao/ElasticsearchMetaAlertDaoTest.java | 27 ++- .../dao/ElasticsearchUpdateDaoTest.java | 52 + .../apache/metron/indexing/dao/HBaseDao.java| 31 +-- .../metron/indexing/dao/MultiIndexDao.java | 138 - .../dao/metaalert/MetaAlertCreateResponse.java | 40 .../dao/metaalert/MetaAlertUpdateDao.java | 20 +- .../AbstractLuceneMetaAlertUpdateDao.java | 81 +++- .../metron/indexing/dao/update/UpdateDao.java | 25 ++- .../InMemoryMetaAlertRetrieveLatestDao.java | 49 + .../metron/indexing/dao/HBaseDaoTest.java | 41 .../apache/metron/indexing/dao/InMemoryDao.java | 18 +- .../indexing/dao/InMemoryMetaAlertDao.java | 199 +-- .../dao/InMemoryMetaAlertUpdateDao.java | 91 + .../metron/indexing/dao/MultiIndexDaoTest.java | 96 + .../metron/indexing/dao/UpdateDaoTest.java | 74 +++ .../indexing/dao/UpdateIntegrationTest.java | 138 +++-- .../dao/metaalert/MetaAlertIntegrationTest.java | 119 --- .../AbstractLuceneMetaAlertUpdateDaoTest.java | 56 -- .../integration/HBaseDaoIntegrationTest.java| 7 +- .../org/apache/metron/solr/dao/SolrDao.java | 28 +-- .../metron/solr/dao/SolrMetaAlertDao.java | 49 +++-- .../metron/solr/dao/SolrMetaAlertUpdateDao.java | 38 ++-- .../apache/metron/solr/dao/SolrUpdateDao.java | 36 ++-- .../metron/solr/dao/SolrMetaAlertDaoTest.java | 21 +- .../metron/solr/dao/SolrUpdateDaoTest.java | 19 +- 37 files changed, 1206 insertions(+), 727 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/README.md -- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 2c216d1..2ce9522 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -336,6 +336,8 @@ Request and Response objects are JSON formatted. The JSON schemas are available | [ `GET /api/v1/storm/supervisors`](#get-apiv1stormsupervisors)| | [ `PATCH /api/v1/update/patch`](#patch-apiv1updatepatch)| | [ `PUT /api/v1/update/replace`](#put-apiv1updatereplace)| +| [ `POST /api/v1/update/add/comment`](#put-apiv1updateaddcomment)| +| [ `POST /api/v1/update/remove/comment`](#put-apiv1updateremovecomment)| | [ `GET /api/v1/user`](#get-apiv1user)| ### `POST /api/v1/alerts/ui/escalate` @@ -963,6 +965,20 @@ Request and Response objects are JSON formatted. The JSON schemas are available ``` * Returns: * 200 - Current user + +### `POST /api/v1/update/add/comment` + * Description: Add a comment to an alert + * Input: +* request - Comment add request + * Returns: +* 200 - Returns the complete alert document with comments added. + +### `POST /api/v1/update/remove/comment` + * Description: Remove a comment from an alert + * Input: +* request - Comment remove request + * Returns: +* 200 - Returns the complete alert document with comments removed. ### `GET /api/v1/user` * Description: Retrieves the current user http://git-wip-us.apache.org/repos/asf/metron/blob/de533063/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/MetaAlertController.java -- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/MetaAlertController.java
metron git commit: METRON-1756 REST tests should use Embedded LDAP in metron-security (merrimanr) closes apache/metron#1186
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1663-knoxsso 54880ba8f -> 1d927c975
METRON-1756 REST tests should use Embedded LDAP in metron-security (merrimanr)
closes apache/metron#1186
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1d927c97
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1d927c97
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1d927c97
Branch: refs/heads/feature/METRON-1663-knoxsso
Commit: 1d927c975677424bc073838df6d906c3e3026bb5
Parents: 54880ba
Author: merrimanr
Authored: Wed Sep 12 12:14:11 2018 -0500
Committer: rmerriman
Committed: Wed Sep 12 12:14:11 2018 -0500
--
metron-interface/metron-rest/pom.xml| 15 +++
.../src/main/resources/application-test.yml | 17 +---
.../metron/rest/config/TestSecurityConfig.java | 44
.../AlertsUIControllerIntegrationTest.java | 3 +-
.../GlobalConfigControllerIntegrationTest.java | 3 +-
.../GrokControllerIntegrationTest.java | 3 +-
.../HdfsControllerIntegrationTest.java | 3 +-
.../KafkaControllerIntegrationTest.java | 3 +-
.../MetaAlertControllerIntegrationTest.java | 3 +-
.../SearchControllerIntegrationTest.java| 3 +-
...richmentConfigControllerIntegrationTest.java | 3 +-
...IndexingConfigControllerIntegrationTest.java | 3 +-
...orParserConfigControllerIntegrationTest.java | 3 +-
.../StellarControllerIntegrationTest.java | 3 +-
.../StormControllerIntegrationTest.java | 3 +-
.../UpdateControllerIntegrationTest.java| 3 +-
.../UserControllerIntegrationTest.java | 3 +-
.../src/test/resources/log4j.properties | 5 ++-
.../src/test/resources/schema.ldif | 22 ++
19 files changed, 70 insertions(+), 75 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/1d927c97/metron-interface/metron-rest/pom.xml
--
diff --git a/metron-interface/metron-rest/pom.xml
b/metron-interface/metron-rest/pom.xml
index d9d4bfe..223f873 100644
--- a/metron-interface/metron-rest/pom.xml
+++ b/metron-interface/metron-rest/pom.xml
@@ -372,6 +372,21 @@
org.eclipse.persistence.jpa
${eclipse.link.version}
+
+
+org.apache.directory.server
+apacheds-all
+1.5.4
+test
+
+
+
+org.apache.metron
+metron-ui-security
+${project.parent.version}
+test
+test-jar
+
http://git-wip-us.apache.org/repos/asf/metron/blob/1d927c97/metron-interface/metron-rest/src/main/resources/application-test.yml
--
diff --git
a/metron-interface/metron-rest/src/main/resources/application-test.yml
b/metron-interface/metron-rest/src/main/resources/application-test.yml
index e6532fa..1202da7 100644
--- a/metron-interface/metron-rest/src/main/resources/application-test.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-test.yml
@@ -62,19 +62,4 @@ meta:
knox:
sso:
-url:
-
-ldap:
- provider:
-url: ldap://localhost:33389
-userdn: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org
-password: password
- user:
-dn.patterns: uid={0},ou=people,dc=hadoop,dc=apache,dc=org
-passwordAttribute: userPassword
-searchBase: ou=people,dc=hadoop,dc=apache,dc=org
-searchFilter: ""
- group:
-searchBase: ou=groups,dc=hadoop,dc=apache,dc=org
-searchFilter: "member={0}"
-roleAttribute: "cn"
+url:
http://git-wip-us.apache.org/repos/asf/metron/blob/1d927c97/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestSecurityConfig.java
--
diff --git
a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestSecurityConfig.java
b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestSecurityConfig.java
deleted file mode 100644
index 04e82b9..000
---
a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestSecurityConfig.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless require
[2/3] metron git commit: METRON-1665 Move hosting of Alerts and Config UIs from Nodejs to Spring Boot (simonellistonball via merrimanr) closes apache/metron#1111
http://git-wip-us.apache.org/repos/asf/metron/blob/54880ba8/metron-interface/metron-rest/README.md -- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 44594f7..42c3d0b 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -25,7 +25,6 @@ This module provides a RESTful API for interacting with Metron. * A running real-time store, either Elasticsearch or Solr depending on which one is enabled * Java 8 installed * Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed -* A relational database ## Installation @@ -66,10 +65,6 @@ No optional parameter has a default. | Environment Variable | Description | - | --- -| METRON_JDBC_DRIVER| JDBC driver class -| METRON_JDBC_URL | JDBC url -| METRON_JDBC_USERNAME | JDBC username -| METRON_JDBC_PLATFORM | JDBC platform (one of h2, mysql, postgres, oracle | ZOOKEEPER | Zookeeper quorum (ex. node1:2181,node2:2181) | BROKERLIST| Kafka Broker list (ex. node1:6667,node2:6667) | HDFS_URL | HDFS url or `fs.defaultFS` Hadoop setting (ex. hdfs://node1:8020) @@ -80,7 +75,6 @@ No optional parameter has a default. | METRON_LOG_DIR| Directory where the log file is written | Optional | /var/log/metron/ | METRON_PID_FILE | File where the pid is written | Optional | /var/run/metron/ | METRON_REST_PORT | REST application port | Optional | 8082 -| METRON_JDBC_CLIENT_PATH | Path to JDBC client jar | Optional | H2 is bundled | METRON_TEMP_GROK_PATH | Temporary directory used to test grok statements | Optional | ./patterns/temp | METRON_DEFAULT_GROK_PATH | Defaults HDFS directory used to store grok statements | Optional | /apps/metron/patterns | SECURITY_ENABLED | Enables Kerberos support | Optional | false @@ -96,27 +90,6 @@ No optional parameter has a default. These are set in the `/etc/default/metron` file. -## Database setup - -The REST application persists data in a relational database and requires a dedicated database user and database (see https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html for more detail). -Spring uses Hibernate as the default ORM framework but another framework is needed becaused Hibernate is not compatible with the Apache 2 license. For this reason Metron uses [EclipseLink](https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html#boot-features-embedded-database-support). See the [Spring Data JPA - EclipseLink](https://github.com/spring-projects/spring-data-examples/tree/master/jpa/eclipselink) project for an example on how to configure EclipseLink in Spring. - -### Development - -The REST application comes with [embedded database support](https://docs.spring.io/spring-boot/docs/current/reference/html/boot-features-sql.html#boot-features-embedded-database-support) for development purposes. - -For example, edit these variables in `/etc/default/metron` before starting the application to configure H2: -``` -METRON_JDBC_DRIVER="org.h2.Driver" -METRON_JDBC_URL="jdbc:h2:file:~/metrondb" -METRON_JDBC_USERNAME="root" -METRON_JDBC_PLATFORM="h2" -``` - -### Production - -The REST application should be configured with a production-grade database outside of development. - Ambari Install Installing with Ambari is recommended for production deployments. @@ -125,48 +98,17 @@ This includes managing the PID file, directing logging, etc. Manual Install -The following configures the application for MySQL: - -1. Install MySQL if not already available (this example uses version 5.7, installation instructions can be found [here](https://dev.mysql.com/doc/refman/5.7/en/linux-installation-yum-repo.html)) - -1. Create a metron user and REST database and permission the user for that database: -``` -CREATE USER 'metron'@'node1' IDENTIFIED BY 'Myp@ssw0rd'; -CREATE DATABASE IF NOT EXISTS metronrest; -GRANT ALL PRIVILEGES ON metronrest.* TO 'metron'@'node1'; -``` - -1. Create the security tables as described in the [Spring Security Guide](https://docs.spring.io/spring-security/site/docs/5.0.4.RELEASE/reference/htmlsingle/#user-schema). - -1. Install the MySQL JDBC client onto the REST application host and configurate the
[3/3] metron git commit: METRON-1665 Move hosting of Alerts and Config UIs from Nodejs to Spring Boot (simonellistonball via merrimanr) closes apache/metron#1111
METRON-1665 Move hosting of Alerts and Config UIs from Nodejs to Spring Boot (simonellistonball via merrimanr) closes apache/metron# Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/54880ba8 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/54880ba8 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/54880ba8 Branch: refs/heads/feature/METRON-1663-knoxsso Commit: 54880ba8f2467fefe56b9c377da003d08f0fe203 Parents: 28f4b57 Author: simonellistonball Authored: Tue Aug 28 14:20:10 2018 -0500 Committer: rmerriman Committed: Tue Aug 28 14:20:10 2018 -0500 -- .gitignore | 6 + dependencies_with_url.csv | 48 +++ .../packaging/ambari/metron-mpack/README.md | 2 - .../common-services/METRON/CURRENT/metainfo.xml | 3 - .../packaging/docker/ansible-docker/Dockerfile | 3 - .../packaging/docker/deb-docker/Dockerfile | 4 - .../docker/deb-docker/prepackage/metron-config | 3 - .../packaging/docker/rpm-docker/Dockerfile | 4 - .../docker/rpm-docker/SPECS/metron.spec | 97 ++- .../packaging/docker/rpm-docker/pom.xml | 4 +- metron-interface/metron-alerts-host/.gitignore | 24 ++ metron-interface/metron-alerts-host/pom.xml | 141 + .../src/main/assembly/assembly.xml | 39 +++ .../org/apache/metron/ui/AlertsApplication.java | 29 ++ .../src/main/resources/application.yml | 61 .../src/main/scripts/metron-alerts.sh | 55 metron-interface/metron-alerts/.gitignore | 3 + metron-interface/metron-alerts/assembly.xml | 55 metron-interface/metron-alerts/pom.xml | 21 +- .../metron-alerts/scripts/alerts-server.js | 76 - .../metron-alerts/scripts/metron-alerts-ui | 159 -- .../metron-alerts/scripts/package.json | 21 -- metron-interface/metron-config-host/pom.xml | 142 + .../src/main/assembly/assembly.xml | 39 +++ .../org/apache/metron/ui/ConfigApplication.java | 31 ++ .../src/main/resources/application.yml | 34 +++ .../src/main/scripts/metron-config.sh | 55 metron-interface/metron-config/.gitignore | 3 + metron-interface/metron-config/assembly.xml | 65 - metron-interface/metron-config/pom.xml | 236 --- .../metron-config/scripts/metron-management-ui | 159 -- .../metron-config/scripts/package.json | 22 -- .../metron-config/scripts/server.js | 76 - metron-interface/metron-rest/README.md | 95 ++ metron-interface/metron-rest/pom.xml| 6 +- .../metron/rest/MetronRestApplication.java | 16 +- .../metron/rest/config/WebSecurityConfig.java | 108 --- .../rest/controller/AlertsUIController.java | 6 +- .../src/main/resources/application-test.yml | 19 ++ .../metron-rest/src/main/scripts/metron-rest.sh | 7 +- .../metron/rest/config/TestSecurityConfig.java | 44 +++ .../AlertsUIControllerIntegrationTest.java | 18 +- metron-interface/metron-ui-host/pom.xml | 138 + .../metron/ui/AbstractHostApplication.java | 28 ++ .../org/apache/metron/ui/UserController.java| 71 + .../metron/ui/ZuulAuthenticationFilter.java | 77 + .../java/org/apache/metron/ui/ZuulError.java| 34 +++ .../org/apache/metron/ui/ZuulErrorFilter.java | 67 + .../src/main/resources/application.yml | 59 .../apache/metron/ui/TestHostApplication.java | 25 ++ .../java/org/apache/metron/ui/WhoamiTest.java | 120 .../ui/ZuulAuthorizationHeaderProxyTest.java| 103 +++ .../src/test/resources/application-test.yml | 77 + metron-interface/metron-ui-security/pom.xml | 135 + .../metron/ui/KnoxSSOAuthenticationFilter.java | 289 +++ .../ui/MetronAuthenticationException.java | 29 ++ .../metron/ui/MetronAuthenticationProvider.java | 60 .../apache/metron/ui/MetronSecurityConfig.java | 188 .../java/org/apache/metron/ui/EmbeddedLdap.java | 164 +++ .../java/org/apache/metron/ui/JWTTests.java | 170 +++ .../ui/KnoxSSOAuthenticationFilterTests.java| 66 + .../ui/MetronAuthenticationProviderTests.java | 33 +++ .../metron/ui/config/TestSecurityConfig.java| 31 ++ .../resources/application-embedded-ldap.yml | 35 +++ .../resources/org/apache/metron/ui/headers.pem | 27 ++ .../resources/org/apache/metron/ui/invalid.pem | 27 ++ .../org/apache/metron/ui/noheaders.pem | 25 ++ .../src/test/resources/schema.ldif | 77 + metron-interface/pom.xml| 13 + 69 files changed, 3147 insertions(+), 1060 deletions
[1/3] metron git commit: METRON-1665 Move hosting of Alerts and Config UIs from Nodejs to Spring Boot (simonellistonball via merrimanr) closes apache/metron#1111
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1663-knoxsso 28f4b5704 -> 54880ba8f
http://git-wip-us.apache.org/repos/asf/metron/blob/54880ba8/metron-interface/metron-ui-security/src/main/java/org/apache/metron/ui/MetronAuthenticationProvider.java
--
diff --git
a/metron-interface/metron-ui-security/src/main/java/org/apache/metron/ui/MetronAuthenticationProvider.java
b/metron-interface/metron-ui-security/src/main/java/org/apache/metron/ui/MetronAuthenticationProvider.java
new file mode 100644
index 000..42d8a2d
--- /dev/null
+++
b/metron-interface/metron-ui-security/src/main/java/org/apache/metron/ui/MetronAuthenticationProvider.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.ui;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.springframework.security.authentication.AuthenticationProvider;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.AuthenticationException;
+import org.springframework.security.core.GrantedAuthority;
+import org.springframework.security.core.authority.SimpleGrantedAuthority;
+
+public class MetronAuthenticationProvider implements AuthenticationProvider {
+
+ @Override
+ public Authentication authenticate(Authentication authentication) throws
AuthenticationException {
+if (authentication != null) {
+ authentication = getSSOAuthentication(authentication);
+ if (authentication != null && authentication.isAuthenticated()) {
+return authentication;
+ }
+}
+throw new MetronAuthenticationException("Authentication failed");
+ }
+
+ private Authentication getSSOAuthentication(Authentication authentication) {
+return authentication;
+ }
+
+ @Override
+ public boolean supports(Class authentication) {
+return true;
+ }
+
+ public static List getAuthoritiesFromUGI(String userName) {
+// TODO - if we have ldap, we can lookup groups for this user
+
+// TODO - if we have a default mapper we can use that
+
+List grantedAuths = new ArrayList();
+grantedAuths.add(new SimpleGrantedAuthority("USER"));
+return grantedAuths;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/54880ba8/metron-interface/metron-ui-security/src/main/java/org/apache/metron/ui/MetronSecurityConfig.java
--
diff --git
a/metron-interface/metron-ui-security/src/main/java/org/apache/metron/ui/MetronSecurityConfig.java
b/metron-interface/metron-ui-security/src/main/java/org/apache/metron/ui/MetronSecurityConfig.java
new file mode 100644
index 000..7d3ec3b
--- /dev/null
+++
b/metron-interface/metron-ui-security/src/main/java/org/apache/metron/ui/MetronSecurityConfig.java
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.ui;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.interfaces.RSAPublicKey;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.HttpMethod;
+import
[3/3] metron git commit: METRON-1752 Prevent package.lock from changing during build (sardell via merrimanr) closes apache/metron#1177
METRON-1752 Prevent package.lock from changing during build (sardell via merrimanr) closes apache/metron#1177 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/09826da8 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/09826da8 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/09826da8 Branch: refs/heads/master Commit: 09826da85eddb014a4b718233d53480220b1f5dd Parents: e66cfc8 Author: sardell Authored: Mon Aug 27 15:14:19 2018 -0500 Committer: rmerriman Committed: Mon Aug 27 15:14:19 2018 -0500 -- metron-interface/metron-alerts/pom.xml | 6 +- .../metron-config/package-lock.json | 17734 + metron-interface/metron-config/pom.xml |30 +- 3 files changed, 17752 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/09826da8/metron-interface/metron-alerts/pom.xml -- diff --git a/metron-interface/metron-alerts/pom.xml b/metron-interface/metron-alerts/pom.xml index 996f473..f67a013 100644 --- a/metron-interface/metron-alerts/pom.xml +++ b/metron-interface/metron-alerts/pom.xml @@ -26,7 +26,7 @@ UTF-8 UTF-8 v9.11.1 -5.8.0 +6.2.0 @@ -52,12 +52,12 @@ generate-resources -npm install +npm ci npm -install +ci
[1/3] metron git commit: METRON-1752 Prevent package.lock from changing during build (sardell via merrimanr) closes apache/metron#1177
Repository: metron Updated Branches: refs/heads/master e66cfc80e -> 09826da85 http://git-wip-us.apache.org/repos/asf/metron/blob/09826da8/metron-interface/metron-config/pom.xml -- diff --git a/metron-interface/metron-config/pom.xml b/metron-interface/metron-config/pom.xml index 14acdc6..59c416e 100644 --- a/metron-interface/metron-config/pom.xml +++ b/metron-interface/metron-config/pom.xml @@ -1,15 +1,15 @@ - http://maven.apache.org/POM/4.0.0; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; @@ -25,8 +25,8 @@ UTF-8 UTF-8 -v6.2.0 -3.8.9 +v9.11.1 +6.2.0 @@ -53,12 +53,12 @@ generate-resources -npm install +npm ci npm - install + ci
[2/3] metron git commit: METRON-1752 Prevent package.lock from changing during build (sardell via merrimanr) closes apache/metron#1177
http://git-wip-us.apache.org/repos/asf/metron/blob/09826da8/metron-interface/metron-config/package-lock.json
--
diff --git a/metron-interface/metron-config/package-lock.json
b/metron-interface/metron-config/package-lock.json
new file mode 100644
index 000..e49bc4a
--- /dev/null
+++ b/metron-interface/metron-config/package-lock.json
@@ -0,0 +1,17734 @@
+{
+ "name": "metron-management-ui",
+ "version": "0.5.1",
+ "lockfileVersion": 1,
+ "requires": true,
+ "dependencies": {
+"@angular-cli/ast-tools": {
+ "version": "1.0.16",
+ "resolved":
"https://registry.npmjs.org/@angular-cli/ast-tools/-/ast-tools-1.0.16.tgz;,
+ "integrity": "sha1-YxmULBol+4TjKUID6fejJmMvzlA=",
+ "dev": true,
+ "requires": {
+"@angular/tsc-wrapped": "0.5.2",
+"denodeify": "1.2.1",
+"rxjs": "5.1.0",
+"typescript": "2.0.10"
+ },
+ "dependencies": {
+"@angular/tsc-wrapped": {
+ "version": "0.5.2",
+ "resolved":
"https://registry.npmjs.org/@angular/tsc-wrapped/-/tsc-wrapped-0.5.2.tgz;,
+ "integrity": "sha1-Lt30csRn/LM06pTe3aqnGZDFpII=",
+ "dev": true,
+ "requires": {
+"tsickle": "0.2.5"
+ }
+},
+"denodeify": {
+ "version": "1.2.1",
+ "resolved":
"https://registry.npmjs.org/denodeify/-/denodeify-1.2.1.tgz;,
+ "integrity": "sha1-OjYof1A05pnnV3kBBSwubJQlFjE=",
+ "dev": true
+},
+"minimist": {
+ "version": "1.2.0",
+ "resolved":
"https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz;,
+ "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
+ "dev": true
+},
+"mkdirp": {
+ "version": "0.5.1",
+ "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz;,
+ "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+ "dev": true,
+ "requires": {
+"minimist": "0.0.8"
+ },
+ "dependencies": {
+"minimist": {
+ "version": "0.0.8",
+ "resolved":
"https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz;,
+ "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
+ "dev": true
+}
+ }
+},
+"rxjs": {
+ "version": "5.1.0",
+ "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-5.1.0.tgz;,
+ "integrity": "sha1-CqkBi39EC1BfpCvXQrZzi+VQ5yA=",
+ "dev": true,
+ "requires": {
+"symbol-observable": "1.0.4"
+ }
+},
+"source-map": {
+ "version": "0.5.6",
+ "resolved":
"https://registry.npmjs.org/source-map/-/source-map-0.5.6.tgz;,
+ "integrity": "sha1-dc449SvwczxafwwRjYEzSiu19BI=",
+ "dev": true
+},
+"source-map-support": {
+ "version": "0.4.11",
+ "resolved":
"https://registry.npmjs.org/source-map-support/-/source-map-support-0.4.11.tgz;,
+ "integrity": "sha1-ZH+TmXizhTWQlTCIUwPa8jJ58yI=",
+ "dev": true,
+ "requires": {
+"source-map": "0.5.6"
+ }
+},
+"symbol-observable": {
+ "version": "1.0.4",
+ "resolved":
"https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.0.4.tgz;,
+ "integrity": "sha1-Kb9hXUqnEhvdiYsi1LP5vE4qoD0=",
+ "dev": true
+},
+"tsickle": {
+ "version": "0.2.5",
+ "resolved": "https://registry.npmjs.org/tsickle/-/tsickle-0.2.5.tgz;,
+ "integrity": "sha1-YNjhJGLm+PvayS1fX+rSv0kIXYI=",
+ "dev": true,
+ "requires": {
+"minimist": "1.2.0",
+"mkdirp": "0.5.1",
+"source-map": "0.5.6",
+"source-map-support": "0.4.11"
+ }
+},
+"typescript": {
+ "version": "2.0.10",
+ "resolved":
"https://registry.npmjs.org/typescript/-/typescript-2.0.10.tgz;,
+ "integrity": "sha1-zN1O2G/VVQpAcQGggUAS4bP6w90=",
+ "dev": true
+}
+ }
+},
+"@angular-cli/base-href-webpack": {
+ "version": "1.0.16",
+ "resolved":
"https://registry.npmjs.org/@angular-cli/base-href-webpack/-/base-href-webpack-1.0.16.tgz;,
+ "integrity": "sha1-Qpai/324TdwuZ8KhB+J29yRj/40=",
+ "dev": true
+},
+"@angular/common": {
+ "version": "2.0.0",
+ "resolved":
"https://registry.npmjs.org/@angular/common/-/common-2.0.0.tgz;,
+ "integrity": "sha1-bVs4zvi+xpcmWQ85XaEMLYkUYkc="
+},
+"@angular/compiler": {
+ "version": "2.0.0",
+ "resolved":
"https://registry.npmjs.org/@angular/compiler/-/compiler-2.0.0.tgz;,
+ "integrity": "sha1-ck58aKwXeqkzqnWmtMsR5ap5XSw="
+},
+"@angular/compiler-cli": {
+ "version": "0.6.4",
+ "resolved":
[21/51] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/app-routing.module.ts
--
diff --git a/metron-interface/metron-alerts/src/app/app-routing.module.ts
b/metron-interface/metron-alerts/src/app/app-routing.module.ts
index 8357d66..56ad41c 100644
--- a/metron-interface/metron-alerts/src/app/app-routing.module.ts
+++ b/metron-interface/metron-alerts/src/app/app-routing.module.ts
@@ -26,7 +26,8 @@ const routes: Routes = [
{ path: 'alerts-list', loadChildren:
'app/alerts/alerts-list/alerts-list.module#AlertsListModule', canActivate:
[AuthGuard]},
{ path: 'save-search', loadChildren:
'app/alerts/save-search/save-search.module#SaveSearchModule', canActivate:
[AuthGuard]},
{ path: 'saved-searches', loadChildren:
'app/alerts/saved-searches/saved-searches.module.ts#SavedSearchesModule',
- canActivate: [AuthGuard]}
+ canActivate: [AuthGuard]},
+ { path: 'pcap', loadChildren: 'app/pcap/pcap.module#PcapModule',
canActivate: [AuthGuard] }
];
@NgModule({
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/app.component.html
--
diff --git a/metron-interface/metron-alerts/src/app/app.component.html
b/metron-interface/metron-alerts/src/app/app.component.html
index 8867a9b..1b992ca 100644
--- a/metron-interface/metron-alerts/src/app/app.component.html
+++ b/metron-interface/metron-alerts/src/app/app.component.html
@@ -16,6 +16,14 @@
+
+
+Alerts
+
+
+PCAP
+
+
Logged in as {{authService.currentUser}} -
Logout
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/app.component.scss
--
diff --git a/metron-interface/metron-alerts/src/app/app.component.scss
b/metron-interface/metron-alerts/src/app/app.component.scss
index 8358e7f..8505ee8 100644
--- a/metron-interface/metron-alerts/src/app/app.component.scss
+++ b/metron-interface/metron-alerts/src/app/app.component.scss
@@ -27,6 +27,24 @@
max-height: 50px;
}
+.nav-link
+{
+ padding-bottom: 0;
+ padding-top: 0;
+ color: inherit;
+}
+
+.nav-item.active
+{
+ border-bottom: 3px solid #32abe2;
+ margin-bottom: 5px;
+}
+
+.nav-link.active
+{
+ color: #ff;
+}
+
.logout {
padding-left: 10px;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/app.module.ts
--
diff --git a/metron-interface/metron-alerts/src/app/app.module.ts
b/metron-interface/metron-alerts/src/app/app.module.ts
index 227eaab..82b4970 100644
--- a/metron-interface/metron-alerts/src/app/app.module.ts
+++ b/metron-interface/metron-alerts/src/app/app.module.ts
@@ -46,8 +46,7 @@ import {MetaAlertService} from './service/meta-alert.service';
import {MetaAlertsModule} from './alerts/meta-alerts/meta-alerts.module';
import {SearchService} from './service/search.service';
import { GlobalConfigService } from './service/global-config.service';
-
-
+import {PcapModule} from './pcap/pcap.module';
export function initConfig(config: ColumnNamesService) {
return () => config.list();
@@ -70,7 +69,8 @@ export function initConfig(config: ColumnNamesService) {
ConfigureRowsModule,
SaveSearchModule,
SavedSearchesModule,
-SwitchModule
+SwitchModule,
+PcapModule
],
providers: [{ provide: APP_INITIALIZER, useFactory: initConfig, deps:
[ColumnNamesService], multi: true },
{ provide: DataSource, useClass: ElasticSearchLocalstorageImpl },
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
new file mode 100644
index 000..3a27714
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing software
+ * distributed under the License is distributed on an "AS
[26/51] [abbrv] metron git commit: METRON-1676 PCAP UI - Add data range selector to the filter bar (tiborm via merrimanr) closes apache/metron#1119
METRON-1676 PCAP UI - Add data range selector to the filter bar (tiborm via
merrimanr) closes apache/metron#1119
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/09a7f189
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/09a7f189
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/09a7f189
Branch: refs/heads/master
Commit: 09a7f18994dc8254ca158139c44258cb7f1107ec
Parents: b445bfe
Author: tiborm
Authored: Mon Aug 6 09:14:54 2018 -0500
Committer: rmerriman
Committed: Mon Aug 6 09:14:54 2018 -0500
--
.../pcap-filters/pcap-filters.component.html| 30 ++--
.../pcap-filters/pcap-filters.component.scss| 2 +-
.../pcap-filters/pcap-filters.component.spec.ts | 156 ++-
.../pcap/pcap-filters/pcap-filters.component.ts | 19 ++-
.../metron-alerts/src/app/pcap/pcap.module.ts | 6 +-
.../src/app/pcap/service/pcap.service.ts| 4 +-
.../shared/directives/alert-search.directive.ts | 2 +-
7 files changed, 195 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/09a7f189/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
index abc4cf0..f4133df 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
@@ -12,23 +12,33 @@
the specific language governing permissions and limitations under the
License.
-->
+
+
+From
+
+
+
+To
+
+
+
-IP Source Address
-
+IP Source Address
+
-IP Source Port
-
+IP Source Port
+
- IP Dest Address
-
+ IP Dest Address
+
-IP Dest Port
-
+IP Dest Port
+
@@ -37,8 +47,8 @@
-Include Reverse Traffic
-
+Include Reverse Traffic
+
http://git-wip-us.apache.org/repos/asf/metron/blob/09a7f189/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.scss
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.scss
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.scss
index 80bac08..b33e804 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.scss
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.scss
@@ -28,7 +28,7 @@
}
.form-group {
- padding-right: 0.75em;
+ padding: 0 0.75em 0.75em 0;
label {
margin-right: 1em;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/09a7f189/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
index 8134f1d..c0f9c3b 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -15,10 +16,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+import { async, ComponentFixture, TestBed, fakeAsync, tick } from
'@angular/core/testing';
+import { By } from '@angular/platform-browser';
import { PcapFiltersComponent } from './pcap-filters.component';
import { FormsModule } from '../../../../node_modules/@angular/forms';
+import { Component, Input, Output, EventEmitter } from '@angular/core';
+import { PcapRequest } from '../model/pcap.request';
+import { emit } from 'cluster';
+
+@Component({
+ selector: 'app-date-picker',
+ template: '',
+})
+class FakeDatePicker {
+ @Input() date: string;
+ @Output() dateChange = new EventEmitter();
+}
describe('PcapFiltersComponent', () => {
let component: PcapFiltersComponent;
@@ -29,18 +43,148 @@ describe('PcapFiltersComponent', () => {
imports: [
FormsModule
],
- declarations: [ PcapFiltersComponent ]
+ dec
[33/51] [abbrv] metron git commit: METRON-1726: Refactor PcapTopologyIntegrationTest (mmiklavc via mmiklavc) closes apache/metron#1140
METRON-1726: Refactor PcapTopologyIntegrationTest (mmiklavc via mmiklavc)
closes apache/metron#1140
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7967f358
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7967f358
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7967f358
Branch: refs/heads/master
Commit: 7967f358c6c4b8437935c43e54179126e48e248f
Parents: 3e77859
Author: mmiklavc
Authored: Tue Aug 7 15:02:20 2018 -0600
Committer: Michael Miklavcic
Committed: Tue Aug 7 15:02:20 2018 -0600
--
.../org/apache/metron/pcap/query/PcapCli.java | 2 +
.../PcapTopologyIntegrationTest.java| 892 ++-
.../metron/pcap/finalizer/PcapCliFinalizer.java | 5 +-
3 files changed, 473 insertions(+), 426 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/7967f358/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
index 0b06b0c..c23f037 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
@@ -87,6 +87,7 @@ public class PcapCli {
try {
config = fixedParser.parse(otherArgs);
commonConfig = config;
+PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path(execDir));
} catch (ParseException | java.text.ParseException e) {
System.err.println(e.getMessage());
System.err.flush();
@@ -112,6 +113,7 @@ public class PcapCli {
try {
config = queryParser.parse(otherArgs);
commonConfig = config;
+PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path(execDir));
} catch (ParseException | java.text.ParseException e) {
System.err.println(e.getMessage());
queryParser.printHelp();
http://git-wip-us.apache.org/repos/asf/metron/blob/7967f358/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
index 108fd2b..c30267d 100644
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
+++
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
@@ -58,6 +58,7 @@ import org.apache.metron.integration.components.MRComponent;
import org.apache.metron.integration.components.ZKServerComponent;
import org.apache.metron.integration.utils.KafkaUtil;
import org.apache.metron.job.JobStatus;
+import org.apache.metron.job.Pageable;
import org.apache.metron.job.Statusable;
import org.apache.metron.pcap.PacketInfo;
import org.apache.metron.pcap.PcapHelper;
@@ -73,7 +74,10 @@ import org.apache.metron.spout.pcap.Endianness;
import org.apache.metron.spout.pcap.deserializer.Deserializers;
import org.apache.metron.test.utils.UnitTestHelper;
import org.json.simple.JSONObject;
+import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -85,12 +89,19 @@ public class PcapTopologyIntegrationTest extends
BaseIntegrationTest {
private static String OUTPUT_DIR = BASE_DIR + "/output";
private static final int MAX_RETRIES = 30;
private static final int SLEEP_MS = 500;
- private String topologiesDir = "src/main/flux";
- private String targetDir = "target";
+ private static String topologiesDir = "src/main/flux";
+ private static String targetDir = "target";
+ private static ComponentRunner runner;
+ private static File inputDir;
+ private static File interimResultDir;
+ private static File outputDir;
+ private static List> pcapEntries;
+ private static boolean withHeaders;
+ private FixedPcapConfig configuration;
private static void clearOutDirs(File... dirs) throws IOException {
-for(File dir: dirs) {
- for(File f : dir.listFiles()) {
+for (File dir : dirs) {
+ for (File f : dir.listFiles()) {
if (f.isDirectory()) {
FileUtils.deleteDirectory(f);
} else {
@@ -99,8 +110,8 @@ public class PcapTopologyIntegrationTest extends
[31/51] [abbrv] metron git commit: METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr) closes apache/metron#1122
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
index 155e1f8..de39a0a 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
@@ -16,9 +16,10 @@
* limitations under the License.
*/
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+import { fakePacket } from '../model/pdml.mock';
import { PcapPacketComponent } from './pcap-packet.component';
-import { PdmlPacket } from '../model/pdml';
+import { By } from '@angular/platform-browser';
describe('PcapPacketComponent', () => {
let component: PcapPacketComponent;
@@ -34,1250 +35,45 @@ describe('PcapPacketComponent', () => {
beforeEach(() => {
fixture = TestBed.createComponent(PcapPacketComponent);
component = fixture.componentInstance;
-component.packet = fakePacket as PdmlPacket;
+component.packet = fakePacket;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
-});
-const fakePacket = {
- name: "something",
- expanded: false,
- protos: [
-{
- "name": "geninfo",
- "pos": "0",
- "showname": "General information",
- "size": "722",
- "hide": null,
- "fields": [
-{
- "name": "num",
- "pos": "0",
- "showname": "Number",
- "size": "722",
- "value": "1",
- "show": "1",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "len",
- "pos": "0",
- "showname": "Frame Length",
- "size": "722",
- "value": "2d2",
- "show": "722",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "caplen",
- "pos": "0",
- "showname": "Captured Length",
- "size": "722",
- "value": "2d2",
- "show": "722",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "timestamp",
- "pos": "0",
- "showname": "Captured Time",
- "size": "722",
- "value": "1458240269.373968000",
- "show": "Mar 17, 2016 18:44:29.373968000 UTC",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-}
- ]
-},
-{
- "name": "frame",
- "pos": "0",
- "showname": "Frame 1: 722 bytes on wire (5776 bits), 722 bytes captured
(5776 bits)",
- "size": "722",
- "hide": null,
- "fields": [
-{
- "name": "frame.dlt",
- "pos": "0",
- "showname": "WTAP_ENCAP: 1",
- "size": "0",
- "value": null,
- "show": "1",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.time",
- "pos": "0",
- "showname": "Arrival Time: Mar 17, 2016 18:44:29.373968000 UTC",
- "size": "0",
- "value": null,
- "show": "Mar 17, 2016 18:44:29.373968000",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.offset_shift",
- "pos": "0",
- "showname": "Time shift for this packet: 0.0 seconds",
- "size": "0",
- "value": null,
- "show": "0.0",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.time_epoch",
- "pos": "0",
- "showname": "Epoch Time: 1458240269.373968000 seconds",
- "size": "0",
- "value": null,
- "show": "1458240269.373968000",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.time_delta",
- "pos": "0",
- "showname": "Time delta from previous captured frame: 0.0
seconds",
- "size": "0",
- "value": null,
- "show": "0.0",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.time_delta_displayed",
- "pos":
[39/51] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1554-pcap-query-panel
Merge branch 'master' into feature/METRON-1554-pcap-query-panel Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d8a5922b Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d8a5922b Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d8a5922b Branch: refs/heads/master Commit: d8a5922b4cc5b20be7cb08f333a861241c365fd5 Parents: 9064cca 2b0060b Author: Michael Miklavcic Authored: Thu Aug 9 09:31:01 2018 -0600 Committer: Michael Miklavcic Committed: Thu Aug 9 09:31:01 2018 -0600 -- metron-sensors/pycapa/README.md| 92 ++--- metron-sensors/pycapa/requirements.txt | 6 +- 2 files changed, 88 insertions(+), 10 deletions(-) --
[15/51] [abbrv] metron git commit: METRON-1690: Add more context to PcapJob JobStatus (mmiklavc via mmiklavc) closes apache/metron#1128
METRON-1690: Add more context to PcapJob JobStatus (mmiklavc via mmiklavc) closes apache/metron#1128 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9d4842f3 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9d4842f3 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9d4842f3 Branch: refs/heads/master Commit: 9d4842f3d6f355d449d51f3c6ac02f2c904c295d Parents: d0e8757 Author: mmiklavc Authored: Wed Jul 25 08:21:32 2018 -0600 Committer: Michael Miklavcic Committed: Wed Jul 25 08:21:32 2018 -0600 -- .../metron-job_state_statechart_diagram.svg | 18 +- .../metron-job_state_statechart_diagram.xml | 18 +- .../java/org/apache/metron/job/JobStatus.java | 42 - .../org/apache/metron/job/JobStatusTest.java| 55 ++ .../job/manager/InMemoryJobManagerTest.java | 16 ++ .../org/apache/metron/pcap/PcapJobTest.java | 80 +--- .../PcapTopologyIntegrationTest.java| 30 +-- .../java/org/apache/metron/pcap/mr/PcapJob.java | 186 +-- site-book/bin/generate-md.sh| 2 + 9 files changed, 294 insertions(+), 153 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/9d4842f3/metron-platform/metron-job/metron-job_state_statechart_diagram.svg -- diff --git a/metron-platform/metron-job/metron-job_state_statechart_diagram.svg b/metron-platform/metron-job/metron-job_state_statechart_diagram.svg index a99c5ad..c801eae 100644 --- a/metron-platform/metron-job/metron-job_state_statechart_diagram.svg +++ b/metron-platform/metron-job/metron-job_state_statechart_diagram.svg @@ -1,14 +1,14 @@ http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd;> -http://www.w3.org/2000/svg; xmlns:xlink="http://www.w3.org/1999/xlink; width="661px" height="291px" version="1.1" style="background-color: rgb(255, 255, 255);">http://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 92px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-dec oration:inherit;">NOT_RUNNINGNOT_RUNNINGhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 60px; white-space: nowrap; word-wrap: normal; text-align: center;">http://ww w.w3.org/1999/xhtml" style="display:inline-block;text-align:inherit;text-decoration:inherit;">RUNNINGRUNNINGhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 74px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">FINALIZINGFINALIZINGhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 44px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">KILLEDKILLEDhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 44px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">FAILEDFAILEDhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 76px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">SUCCEEDEDSUCCEEDED +http://www.w3.org/2000/svg; xmlns:xlink="http://www.w3.org/1999/xlink; width="656px" height="291px" version="1.1">http://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 92px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml;
[37/51] [abbrv] metron git commit: METRON-1713 PCAP UI - Add a way to kill a pcap job (tiborm via merrimanr) closes apache/metron#1143
METRON-1713 PCAP UI - Add a way to kill a pcap job (tiborm via merrimanr)
closes apache/metron#1143
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/14dcb2d9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/14dcb2d9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/14dcb2d9
Branch: refs/heads/master
Commit: 14dcb2d90581835d8206c65918c24e8cb04bfd06
Parents: 5b3e2c3
Author: tiborm
Authored: Thu Aug 9 08:33:21 2018 -0500
Committer: rmerriman
Committed: Thu Aug 9 08:33:21 2018 -0500
--
.../pcap/pcap-panel/pcap-panel.component.html | 7 +-
.../pcap/pcap-panel/pcap-panel.component.scss | 33 +
.../pcap-panel/pcap-panel.component.spec.ts | 130 +++
.../app/pcap/pcap-panel/pcap-panel.component.ts | 42 +-
.../src/app/pcap/service/pcap.service.ts| 10 +-
5 files changed, 215 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/14dcb2d9/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
index 950f49c..0dda268 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
@@ -15,8 +15,11 @@
-
-{{progressWidth}}%
+
+
+ {{progressWidth}}%
+
+
{{ errorMsg }}
http://git-wip-us.apache.org/repos/asf/metron/blob/14dcb2d9/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
index 8989bf9..523f5ce 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
@@ -31,4 +31,37 @@
.progress-bar {
width: 0;
+ height: 34px;
+ line-height: 34px;
+ vertical-align: middle;
+ font-size: 0.875rem;
+}
+
+.pcap-progress-wrapper {
+ position: relative;
+ padding-right: 55px;
+}
+
+.pcap-cancel-query-button {
+ position: absolute;
+ top: 0;
+ right: 0;
+ padding-top: 6px;
+ padding-bottom: 6px;
+ background: $icon-button-background;
+ min-width: 42px;
+ padding-left: 0;
+ padding-right: 0;
+ border: 1px solid $blue-chill !important;
+ cursor: pointer;
+
+ &:focus {
+box-shadow: none;
+ }
+
+ &::before {
+font-family: "FontAwesome";
+content: '\f00d';
+color: $piction-blue;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/14dcb2d9/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
index 0804b79..9dacc7f 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
@@ -50,6 +50,9 @@ class FakePcapService {
return '';
}
submitRequest() {}
+ cancelQuery() {
+return defer(() => Promise.resolve());
+ }
}
describe('PcapPanelComponent', () => {
@@ -327,6 +330,133 @@ describe('PcapPanelComponent', () => {
expect(fixture.debugElement.query(By.css('app-pcap-list'))).toBeDefined();
}));
+ it('should render a cancel button only if a query runs', () => {
+component.queryRunning = false;
+fixture.detectChanges();
+
expect(fixture.debugElement.query(By.css('[data-qe-id="pcap-cancel-query-button"]'))).toBeFalsy();
+
+component.queryRunning = true;
+fixture.detectChanges();
+
expect(fixture.debugElement.query(By.css('[data-qe-id="pcap-cancel-query-button"]'))).toBeDefined();
+ });
+
+ it('should hide the progress bar if the user clicks on the cancel button',
fakeAsync(() => {
+component.queryRunning = true;
+component.queryId = 'testid';
+fixture.detectChanges();
+expect(fixture.debugElement.query(By.css('.pcap-progress'))).toBeDefined();
+
+const cancelBtn =
fixture.debugElement.query(By.css('[data-qe-id="pcap-cancel-query-button"]'));
+const cancelBtnEl = cancelBtn.nativeElement;
+
+cancelBtnEl.cl
[40/51] [abbrv] metron git commit: METRON-1722 PcapCLI should print progress to stdout (merrimanr) closes apache/metron#1138
METRON-1722 PcapCLI should print progress to stdout (merrimanr) closes
apache/metron#1138
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b29833cc
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b29833cc
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b29833cc
Branch: refs/heads/master
Commit: b29833cc5db4efca2a6e9025042baf705fb347c6
Parents: d8a5922
Author: merrimanr
Authored: Fri Aug 10 08:25:13 2018 -0500
Committer: rmerriman
Committed: Fri Aug 10 08:25:13 2018 -0500
--
metron-platform/metron-pcap-backend/README.md | 2 ++
.../java/org/apache/metron/pcap/query/CliParser.java| 4
.../java/org/apache/metron/pcap/query/PcapCliTest.java | 12 +---
.../java/org/apache/metron/pcap/config/PcapConfig.java | 9 +
.../java/org/apache/metron/pcap/config/PcapOptions.java | 3 ++-
.../main/java/org/apache/metron/pcap/mr/PcapJob.java| 11 +++
6 files changed, 37 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/README.md
--
diff --git a/metron-platform/metron-pcap-backend/README.md
b/metron-platform/metron-pcap-backend/README.md
index e7960e3..2ff20d8 100644
--- a/metron-platform/metron-pcap-backend/README.md
+++ b/metron-platform/metron-pcap-backend/README.md
@@ -132,6 +132,7 @@ usage: Fixed filter options
-nr,--num_reducers The number of reducers to use. Default
is 10.
-h,--help Display help
+ -ps,--print_status Print the status of the job as it runs
-ir,--include_reverse Indicates if filter should check swapped
src/dest addresses and IPs
-p,--protocol IP Protocol
@@ -154,6 +155,7 @@ usage: Query filter options
-nr,--num_reducers The number of reducers to use. Default
is 10.
-h,--help Display help
+ -ps,--print_status Print the status of the job as it runs
-q,--query Query string to use as a filter
-st,--start_time (required) Packet start time range.
```
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index 2d15e8b..69c725c 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -55,6 +55,7 @@ public class CliParser {
options.addOption(newOption("rpf", "records_per_file", true,
String.format("Number of records to include in each output pcap file (defaults
to %s)", NUM_RECORDS_PER_FILE_DEFAULT)));
options.addOption(newOption("et", "end_time", true, "Packet end time
range. Default is current system time."));
options.addOption(newOption("df", "date_format", true, "Date format to use
for parsing start_time and end_time. Default is to use time in millis since the
epoch."));
+options.addOption(newOption("ps", "print_status", false, "Print the status
of the job as it runs"));
return options;
}
@@ -125,6 +126,9 @@ public class CliParser {
//no-op
}
}
+if (commandLine.hasOption("print_status")) {
+ config.setPrintJobStatus(true);
+}
}
public void printHelp(String msg, Options opts) {
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index 96ca354..7c75224 100644
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -147,7 +147,8 @@ public class PcapCliTest {
"-protocol", "6",
"-include_reverse",
&
[27/51] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1554-pcap-query-panel
Merge branch 'master' into feature/METRON-1554-pcap-query-panel Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/89726659 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/89726659 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/89726659 Branch: refs/heads/master Commit: 897266598b7984884d6cf8a6a699fe902e792483 Parents: 09a7f18 1b2bdff Author: merrimanr Authored: Mon Aug 6 11:08:48 2018 -0500 Committer: merrimanr Committed: Mon Aug 6 11:08:48 2018 -0500 -- .../METRON/CURRENT/repos/repoinfo.xml | 7 ++ .../packaging/docker/ansible-docker/Dockerfile | 118 --- .../packaging/docker/deb-docker/Dockerfile | 23 ++-- .../packaging/docker/rpm-docker/Dockerfile | 46 ++-- metron-sensors/pycapa/README.md | 47 +--- 5 files changed, 156 insertions(+), 85 deletions(-) --
[35/51] [abbrv] metron git commit: METRON-1712 PCAP UI - Input validation (tiborm via merrimanr) closes apache/metron#1142
METRON-1712 PCAP UI - Input validation (tiborm via merrimanr) closes
apache/metron#1142
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/52de126f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/52de126f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/52de126f
Branch: refs/heads/master
Commit: 52de126fc746181d6aac8ce002d6b814fd1e6cb5
Parents: afb9607
Author: tiborm
Authored: Wed Aug 8 16:36:35 2018 -0500
Committer: rmerriman
Committed: Wed Aug 8 16:36:35 2018 -0500
--
.../src/app/pcap/model/pcap.mock.ts | 4 +-
.../src/app/pcap/model/pcap.request.ts | 4 +-
.../pcap-filters/pcap-filters.component.html| 12 +-
.../pcap-filters/pcap-filters.component.spec.ts | 228 ++-
.../pcap/pcap-filters/pcap-filters.component.ts | 5 +
5 files changed, 232 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/52de126f/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
index c867fe9..bf02da8 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
@@ -22,9 +22,9 @@ export const fakePcapRequest = {
startTimeMs: 0,
endTimeMs: 0,
ipSrcAddr: '0.0.0.0',
- ipSrcPort: '80',
+ ipSrcPort: 80,
ipDstAddr: '0.0.0.0',
- ipDstPort: '80',
+ ipDstPort: 80,
protocol: '*',
packetFilter: '*',
includeReverse: false
http://git-wip-us.apache.org/repos/asf/metron/blob/52de126f/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
index 91c2287..d00a6ac 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
@@ -20,9 +20,9 @@ export class PcapRequest {
startTimeMs: number = 0;
endTimeMs: number = 15;
ipSrcAddr: string = '';
- ipSrcPort: string = '';
+ ipSrcPort: number = 0;
ipDstAddr: string = '';
- ipDstPort: string = '';
+ ipDstPort: number = 0;
protocol: string = '';
packetFilter: string = '';
includeReverse: boolean = false;
http://git-wip-us.apache.org/repos/asf/metron/blob/52de126f/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
index f4133df..039307a 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
@@ -24,21 +24,21 @@
IP Source Address
-
+
-
+
IP Source Port
-
+
IP Dest Address
-
+
IP Dest Port
-
+
@@ -57,6 +57,6 @@
-
+
http://git-wip-us.apache.org/repos/asf/metron/blob/52de126f/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
index c0f9c3b..4336b22 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
@@ -16,20 +16,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { async, ComponentFixture, TestBed, fakeAsync, tick } from
'@angular/core/testing';
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { By } from '@angular/platform-browser';
import { PcapFiltersComponent } from './pcap-filters.component';
import { FormsModule } from '../../../../node_modules/@angular/forms';
-import { Component, Input, Output, EventEmitter } from '@angular/core';
+import { Component, Input, Output, EventEmitter, DebugElement } from
'@angular/core';
import { PcapRequest } from '../model/pcap.request';
-import { emit } from 'cluster';
@Component({
selector
[42/51] [abbrv] metron git commit: METRON-1731: PCAP - Escape colons in output dir names (mmiklavc via mmiklavc) closes apache/metron#1155
METRON-1731: PCAP - Escape colons in output dir names (mmiklavc via mmiklavc)
closes apache/metron#1155
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/73dc63e6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/73dc63e6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/73dc63e6
Branch: refs/heads/master
Commit: 73dc63e671b55d22d251f4be1c217259f4f5dc71
Parents: 05316a4
Author: mmiklavc
Authored: Fri Aug 10 12:42:47 2018 -0600
Committer: Michael Miklavcic
Committed: Fri Aug 10 12:42:47 2018 -0600
--
.../apache/metron/pcap/FixedPcapFilterTest.java | 286 --
.../org/apache/metron/pcap/PcapJobTest.java | 290 ---
.../apache/metron/pcap/QueryPcapFilterTest.java | 228 ---
.../pcap/filter/fixed/FixedPcapFilter.java | 14 +-
.../pcap/filter/query/QueryPcapFilter.java | 17 +-
.../metron/pcap/mr/OutputDirFormatter.java | 37 +++
.../java/org/apache/metron/pcap/mr/PcapJob.java | 5 +-
.../pcap/filter/fixed/FixedPcapFilterTest.java | 271 -
.../pcap/filter/query/QueryPcapFilterTest.java | 207 -
.../metron/pcap/mr/OutputDirFormatterTest.java | 62
.../org/apache/metron/pcap/mr/PcapJobTest.java | 290 +++
11 files changed, 877 insertions(+), 830 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/73dc63e6/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
deleted file mode 100644
index 84969d3..000
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.pcap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.metron.common.Constants;
-import org.apache.metron.pcap.filter.fixed.FixedPcapFilter;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class FixedPcapFilterTest {
- @Test
- public void testTrivialEquality() throws Exception {
-Configuration config = new Configuration();
-final Map fields = new HashMap() {{
- put(Constants.Fields.SRC_ADDR.getName(), "src_ip");
- put(Constants.Fields.SRC_PORT.getName(), "0");
- put(Constants.Fields.DST_ADDR.getName(), "dst_ip");
- put(Constants.Fields.DST_PORT.getName(), "1");
- put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "false");
-}};
-new FixedPcapFilter.Configurator().addToConfig(fields, config);
-{
- FixedPcapFilter filter = new FixedPcapFilter() {
-@Override
-protected Map packetToFields(PacketInfo pi) {
- return new HashMap() {{
-put(Constants.Fields.SRC_ADDR.getName(), "src_ip");
-put(Constants.Fields.SRC_PORT.getName(), 0);
-put(Constants.Fields.DST_ADDR.getName(), "dst_ip");
-put(Constants.Fields.DST_PORT.getName(), 1);
- }};
-}
- };
- filter.configure(config);
- Assert.assertTrue(filter.test(null));
-}
- }
-
- @Test
- public void testReverseTraffic() throws Exception {
-Configuration config = new Configuration();
-final Map fields = new HashMap() {{
- put(Constants.Fields.SRC_ADDR.getName(), "src_ip");
- put(Constants.Fields.SRC_PORT.getName(), "0");
- put(Constants.Fields.DST_ADDR.getName(), "dst_ip");
- put(Constants.Fields.DST_PORT.getName(), "1");
- put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "true");
-}};
-new FixedPcapFilter.Configurator().addToConfig(fields, config);
-{
- FixedPcapFilter filter = new FixedPcapFilter() {
-@Override
-
[45/51] [abbrv] metron git commit: METRON-1733 PCAP UI - PCAP queries don t work on Safari (sardell via merrimanr) closes apache/metron#1158
METRON-1733 PCAP UI - PCAP queries dont work on Safari (sardell via
merrimanr) closes apache/metron#1158
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bce9b903
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bce9b903
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bce9b903
Branch: refs/heads/master
Commit: bce9b903fbd3436ea0c41eb8d75da279e16ce922
Parents: ba0a609
Author: sardell
Authored: Mon Aug 13 16:04:45 2018 -0500
Committer: rmerriman
Committed: Mon Aug 13 16:04:45 2018 -0500
--
.../src/app/pcap/pcap-filters/pcap-filters.component.ts | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/bce9b903/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
index df91ae3..b23a2e2 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
@@ -59,8 +59,8 @@ export class PcapFiltersComponent implements OnInit,
OnChanges {
}
onSubmit() {
-this.model.startTimeMs = new Date(this.startTimeStr).getTime();
-this.model.endTimeMs = new Date(this.endTimeStr).getTime();
+this.model.startTimeMs = moment(this.startTimeStr,
DEFAULT_TIMESTAMP_FORMAT).valueOf();
+this.model.endTimeMs = moment(this.endTimeStr,
DEFAULT_TIMESTAMP_FORMAT).valueOf();
if (this.ipSrcPort !== '') {
this.model.ipSrcPort = +this.ipSrcPort;
} else {
[23/51] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d5eb56a9 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d5eb56a9 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d5eb56a9 Branch: refs/heads/master Commit: d5eb56a967ac7b925d0a843c027a2b0dde66de06 Parents: 7dff4de Author: tiborm Authored: Wed Aug 1 11:31:50 2018 -0500 Committer: rmerriman Committed: Wed Aug 1 11:31:50 2018 -0500 -- .../metron-alerts/package-lock.json | 3719 +- .../metron-alerts/src/app/app-routing.module.ts |3 +- .../metron-alerts/src/app/app.component.html|8 + .../metron-alerts/src/app/app.component.scss| 18 + .../metron-alerts/src/app/app.module.ts |6 +- .../src/app/pcap/model/pcap.request.ts | 29 + .../metron-alerts/src/app/pcap/model/pdml.ts| 50 + .../pcap-filters/pcap-filters.component.html| 52 + .../pcap-filters/pcap-filters.component.scss| 69 + .../pcap-filters/pcap-filters.component.spec.ts | 46 + .../pcap/pcap-filters/pcap-filters.component.ts | 41 + .../app/pcap/pcap-list/pcap-list.component.html | 37 + .../app/pcap/pcap-list/pcap-list.component.scss | 21 + .../pcap/pcap-list/pcap-list.component.spec.ts | 70 + .../app/pcap/pcap-list/pcap-list.component.ts | 39 + .../pcap-packet-line.component.html | 19 + .../pcap-packet-line.component.scss | 17 + .../pcap-packet-line.component.spec.ts | 1283 ++ .../pcap-packet-line.component.ts | 55 + .../pcap/pcap-packet/pcap-packet.component.html | 21 + .../pcap/pcap-packet/pcap-packet.component.scss | 36 + .../pcap-packet/pcap-packet.component.spec.ts | 1283 ++ .../pcap/pcap-packet/pcap-packet.component.ts | 39 + .../pcap/pcap-panel/pcap-panel.component.html | 31 + .../pcap/pcap-panel/pcap-panel.component.scss | 52 + .../pcap-panel/pcap-panel.component.spec.ts | 68 + .../app/pcap/pcap-panel/pcap-panel.component.ts | 75 + .../metron-alerts/src/app/pcap/pcap.module.ts | 50 + .../metron-alerts/src/app/pcap/pcap.routing.ts | 27 + .../src/app/pcap/service/pcap.service.spec.ts | 1752 + .../src/app/pcap/service/pcap.service.ts| 68 + 31 files changed, 7304 insertions(+), 1780 deletions(-) --
[34/51] [abbrv] metron git commit: METRON-1720 Better error messages when there are no results or wireshark is not installed (merrimanr) closes apache/metron#1154
METRON-1720 Better error messages when there are no results or wireshark is not
installed (merrimanr) closes apache/metron#1154
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/afb9607d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/afb9607d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/afb9607d
Branch: refs/heads/master
Commit: afb9607dfbbaf89f06a30823bf668523bb7b6f9a
Parents: 7967f35
Author: merrimanr
Authored: Wed Aug 8 14:11:26 2018 -0500
Committer: rmerriman
Committed: Wed Aug 8 14:11:26 2018 -0500
--
.../pcap-panel/pcap-panel.component.spec.ts | 64
.../app/pcap/pcap-panel/pcap-panel.component.ts | 9 ++-
.../src/app/pcap/service/pcap.service.ts| 3 +-
3 files changed, 73 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/afb9607d/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
index bd732ad..0804b79 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
@@ -26,6 +26,7 @@ import { PcapPagination } from '../model/pcap-pagination';
import { By } from '../../../../node_modules/@angular/platform-browser';
import { PcapRequest } from '../model/pcap.request';
import { defer } from 'rxjs/observable/defer';
+import {RestError} from "../../model/rest-error";
@Component({
selector: 'app-pcap-filters',
@@ -305,6 +306,7 @@ describe('PcapPanelComponent', () => {
);
const pollResponse = new PcapStatusResponse();
+pollResponse.jobStatus = 'SUCCEEDED';
pcapService.pollStatus = jasmine.createSpy('pollStatus').and.returnValue(
defer(() => Promise.resolve(pollResponse))
);
@@ -321,6 +323,68 @@ describe('PcapPanelComponent', () => {
tick();
fixture.detectChanges();
+expect(component.pdml).toEqual(myPdml);
expect(fixture.debugElement.query(By.css('app-pcap-list'))).toBeDefined();
}));
+
+ it('should handle get packet 404', fakeAsync(() => {
+const searchResponse = new PcapStatusResponse();
+searchResponse.jobId = '42';
+
+pcapService.submitRequest =
jasmine.createSpy('submitRequest').and.returnValue(
+defer(() => Promise.resolve(searchResponse))
+);
+
+const pollResponse = new PcapStatusResponse();
+pollResponse.jobStatus = 'SUCCEEDED';
+pcapService.pollStatus = jasmine.createSpy('pollStatus').and.returnValue(
+defer(() => Promise.resolve(pollResponse))
+);
+
+const restError = new RestError();
+restError.responseCode = 404;
+pcapService.getPackets = jasmine.createSpy('getPackets').and.returnValue(
+defer(() => Promise.reject(restError))
+);
+
+component.onSearch(new PcapRequest());
+
+expect(component.errorMsg).toBeFalsy();
+
+tick();
+fixture.detectChanges();
+
+expect(component.errorMsg).toEqual('No results returned');
+ }));
+
+ it('should handle get packet error', fakeAsync(() => {
+const searchResponse = new PcapStatusResponse();
+searchResponse.jobId = '42';
+
+pcapService.submitRequest =
jasmine.createSpy('submitRequest').and.returnValue(
+defer(() => Promise.resolve(searchResponse))
+);
+
+const pollResponse = new PcapStatusResponse();
+pollResponse.jobStatus = 'SUCCEEDED';
+pcapService.pollStatus = jasmine.createSpy('pollStatus').and.returnValue(
+defer(() => Promise.resolve(pollResponse))
+);
+
+const restError = new RestError();
+restError.responseCode = 500;
+restError.message = 'error message';
+pcapService.getPackets = jasmine.createSpy('getPackets').and.returnValue(
+defer(() => Promise.reject(restError))
+);
+
+component.onSearch(new PcapRequest());
+
+expect(component.errorMsg).toBeFalsy();
+
+tick();
+fixture.detectChanges();
+
+expect(component.errorMsg).toEqual('Response message: error message.
Something went wrong retrieving pdml results!');
+ }));
});
http://git-wip-us.apache.org/repos/asf/metron/blob/afb9607d/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
index 82121d4..8e4ced0 1
[22/51] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/package-lock.json
--
diff --git a/metron-interface/metron-alerts/package-lock.json
b/metron-interface/metron-alerts/package-lock.json
index 3ace2b7..7d3ad99 100644
--- a/metron-interface/metron-alerts/package-lock.json
+++ b/metron-interface/metron-alerts/package-lock.json
@@ -1,6 +1,6 @@
{
"name": "metron-alerts",
- "version": "0.4.3",
+ "version": "0.5.1",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@@ -12,66 +12,66 @@
"requires": {
"@ngtools/json-schema": "1.1.0",
"@ngtools/webpack": "1.5.3",
-"autoprefixer": "6.7.7",
-"chalk": "2.3.0",
-"circular-dependency-plugin": "3.0.0",
-"common-tags": "1.5.1",
-"core-object": "3.1.5",
-"css-loader": "0.28.7",
-"cssnano": "3.10.0",
-"denodeify": "1.2.1",
-"diff": "3.2.0",
-"ember-cli-normalize-entity-name": "1.0.0",
-"ember-cli-string-utils": "1.1.0",
-"exports-loader": "0.6.4",
-"extract-text-webpack-plugin": "2.1.2",
-"file-loader": "0.10.1",
-"fs-extra": "4.0.2",
-"get-caller-file": "1.0.2",
-"glob": "7.1.2",
-"heimdalljs": "0.2.5",
-"heimdalljs-logger": "0.1.9",
-"html-webpack-plugin": "2.30.1",
-"inflection": "1.12.0",
-"inquirer": "3.3.0",
-"isbinaryfile": "3.0.2",
-"istanbul-instrumenter-loader": "2.0.0",
-"json-loader": "0.5.7",
-"karma-source-map-support": "1.2.0",
-"less": "2.7.3",
-"less-loader": "4.0.5",
-"license-webpack-plugin": "0.4.3",
-"lodash": "4.17.4",
-"memory-fs": "0.4.1",
-"minimatch": "3.0.4",
-"node-modules-path": "1.0.1",
-"node-sass": "4.5.3",
-"nopt": "4.0.1",
-"opn": "5.1.0",
-"portfinder": "1.0.13",
-"postcss-loader": "1.3.3",
-"postcss-url": "5.1.2",
-"raw-loader": "0.5.1",
-"resolve": "1.3.3",
-"rsvp": "3.6.2",
-"rxjs": "5.4.0",
-"sass-loader": "6.0.6",
-"script-loader": "0.7.2",
-"semver": "5.3.0",
-"silent-error": "1.1.0",
-"source-map-loader": "0.2.3",
-"style-loader": "0.13.2",
-"stylus": "0.54.5",
-"stylus-loader": "3.0.1",
+"autoprefixer": "^6.5.3",
+"chalk": "^2.0.1",
+"circular-dependency-plugin": "^3.0.0",
+"common-tags": "^1.3.1",
+"core-object": "^3.1.0",
+"css-loader": "^0.28.1",
+"cssnano": "^3.10.0",
+"denodeify": "^1.2.1",
+"diff": "^3.1.0",
+"ember-cli-normalize-entity-name": "^1.0.0",
+"ember-cli-string-utils": "^1.0.0",
+"exports-loader": "^0.6.3",
+"extract-text-webpack-plugin": "^2.1.0",
+"file-loader": "^0.10.0",
+"fs-extra": "^4.0.0",
+"get-caller-file": "^1.0.0",
+"glob": "^7.0.3",
+"heimdalljs": "^0.2.4",
+"heimdalljs-logger": "^0.1.9",
+"html-webpack-plugin": "^2.19.0",
+"inflection": "^1.7.0",
+"inquirer": "^3.0.0",
+"isbinaryfile": "^3.0.0",
+"istanbul-instrumenter-loader": "^2.0.0",
+"json-loader": "^0.5.4",
+"karma-source-map-support": "^1.2.0",
+"less": "^2.7.2",
+"less-loader": "^4.0.2",
+"license-webpack-plugin": "^0.4.2",
+"lodash": "^4.11.1",
+"memory-fs": "^0.4.1",
+"minimatch": "^3.0.3",
+"node-modules-path": "^1.0.0",
+"node-sass": "^4.3.0",
+"nopt": "^4.0.1",
+"opn": "~5.1.0",
+"portfinder": "~1.0.12",
+"postcss-loader": "^1.3.3",
+"postcss-url": "^5.1.2",
+"raw-loader": "^0.5.1",
+"resolve": "^1.1.7",
+"rsvp": "^3.0.17",
+"rxjs": "^5.0.1",
+"sass-loader": "^6.0.3",
+"script-loader": "^0.7.0",
+"semver": "^5.1.0",
+"silent-error": "^1.0.0",
+"source-map-loader": "^0.2.0",
+"style-loader": "^0.13.1",
+"stylus": "^0.54.5",
+"stylus-loader": "^3.0.1",
"temp": "0.8.3",
-"typescript": "2.2.2",
-"url-loader": "0.5.9",
-"walk-sync": "0.3.2",
-"webpack": "2.4.1",
-"webpack-dev-middleware": "1.12.2",
-"webpack-dev-server": "2.4.5",
-"webpack-merge": "2.6.1",
-"zone.js": "0.8.18"
+"typescript": ">=2.0.0 <2.4.0",
+"url-loader": "^0.5.7",
+"walk-sync": "^0.3.1",
+"webpack": "~2.4.0",
+"webpack-dev-middleware": "^1.10.2",
+"webpack-dev-server": "~2.4.5",
+"webpack-merge": "^2.4.0",
+"zone.js": "^0.8.14"
},
"dependencies": {
"ansi-styles": {
@@ -80,7 +80,7 @@
"integrity":
[50/51] [abbrv] metron git commit: METRON-1739 UDP packets are not handled (merrimanr) closes apache/metron#1168
METRON-1739 UDP packets are not handled (merrimanr) closes apache/metron#1168
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/14e80b3c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/14e80b3c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/14e80b3c
Branch: refs/heads/master
Commit: 14e80b3c229ea2e5a8d615c87d822d4b44d8ec98
Parents: 076a6a1
Author: merrimanr
Authored: Fri Aug 17 08:47:32 2018 -0500
Committer: rmerriman
Committed: Fri Aug 17 08:47:32 2018 -0500
--
.../src/app/pcap/model/pdml.mock.ts | 31
.../pcap-packet-line.component.spec.ts | 53 --
.../pcap-packet-line.component.ts | 15 ++--
.../java/org/apache/metron/pcap/PcapHelper.java | 18 -
.../org/apache/metron/pcap/PcapHelperTest.java | 76
5 files changed, 180 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/14e80b3c/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
index 6316b92..2520da1 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
@@ -60,3 +60,34 @@ export const fakePacket = {
} as PdmlProto
]
} as PdmlPacket;
+
+export const fakeUdpPacket = {
+ "name": '',
+ "expanded": false,
+ "protos": [
+{
+ "name": "geninfo",
+ "showname": "",
+ "fields": [
+{ "name": "timestamp", "pos": "0", "showname": "Captured Time",
"size": "342", "value": "1534414874.932794000", "show": "Aug 16, 2018
10:21:14.932794000 UTC", "unmaskedvalue": null, "hide": null, "fields": null,
"protos": null } as PdmlField
+ ]
+} as PdmlProto,
+{
+ "name": "ip",
+ "showname": "",
+ "fields": [
+{ "name": "ip.proto", "pos": "23", "showname": "Protocol: UDP (17)",
"size": "1", "value": "11", "show": "17", "unmaskedvalue": null, "hide": null,
"fields": null, "protos": null } as PdmlField,
+{ "name": "ip.src", "pos": "26", "showname": "Source: 0.0.0.0
(0.0.0.0)", "size": "4", "value": "", "show": "0.0.0.0",
"unmaskedvalue": null, "hide": null, "fields": null, "protos": null } as
PdmlField,
+{ "name": "ip.dst", "pos": "30", "showname": "Destination:
255.255.255.255 (255.255.255.255)", "size": "4", "value": "", "show":
"255.255.255.255", "unmaskedvalue": null, "hide": null, "fields": null,
"protos": null } as PdmlField
+ ]
+} as PdmlProto,
+{
+ "name": "udp",
+ "showname": "User Datagram Protocol, Src Port: bootpc (68), Dst Port:
bootps (67)",
+ "fields": [
+{ "name": "udp.srcport", "pos": "34", "showname": "Source port: bootpc
(68)", "size": "2", "value": "0044", "show": "68", "unmaskedvalue": null,
"hide": null, "fields": null, "protos": null } as PdmlField,
+{ "name": "udp.dstport", "pos": "36", "showname": "Destination port:
bootps (67)", "size": "2", "value": "0043", "show": "67", "unmaskedvalue":
null, "hide": null, "fields": null, "protos": null } as PdmlField
+ ],
+} as PdmlProto
+ ]
+} as PdmlPacket;
http://git-wip-us.apache.org/repos/asf/metron/blob/14e80b3c/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-pack
[48/51] [abbrv] metron git commit: METRON-1737: Document Job cleanup (merrimanr via mmiklavc) closes apache/metron#1164
METRON-1737: Document Job cleanup (merrimanr via mmiklavc) closes
apache/metron#1164
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6b70571d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6b70571d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6b70571d
Branch: refs/heads/master
Commit: 6b70571d6de3951c98269bbf5b38e8b69deddfab
Parents: d9e1f38
Author: merrimanr
Authored: Wed Aug 15 16:00:13 2018 -0600
Committer: Michael Miklavcic
Committed: Wed Aug 15 16:00:13 2018 -0600
--
metron-interface/metron-rest/README.md | 11 +++
1 file changed, 11 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/6b70571d/metron-interface/metron-rest/README.md
--
diff --git a/metron-interface/metron-rest/README.md
b/metron-interface/metron-rest/README.md
index 080422d..2c216d1 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -222,6 +222,17 @@ Out of the box it is a simple wrapper around the tshark
command to transform raw
REST will supply the script with raw pcap data through standard in and expects
PDML data serialized as XML.
Pcap query jobs can be configured for submission to a YARN queue. This
setting is exposed as the Spring property `pcap.yarn.queue`. If configured,
the REST application will set the `mapreduce.job.queuename` Hadoop property to
that value.
+It is highly recommended that a dedicated YARN queue be created and configured
for Pcap queries to prevent a job from consuming too many cluster resources.
More information about setting up YARN queues can be found
[here](https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html#Setting_up_queues).
+
+Pcap query results are stored in HDFS. The location of query results when run
through the REST app is determined by a couple factors. The root of Pcap query
results defaults to `/apps/metron/pcap/output` but can be changed with the
+Spring property `pcap.final.output.path`. Assuming the default Pcap query
output directory, the path to a result page will follow this pattern:
+```
+/apps/metron/pcap/output/{username}/MAP_REDUCE/{job id}/page-{page number}.pcap
+```
+Over time Pcap query results will accumulate in HDFS. Currently these results
are not cleaned up automatically so cluster administrators should be aware of
this and monitor them. It is highly recommended that a process be put in place
to
+periodically delete files and directories under the Pcap query results root.
+
+Users should also be mindful of date ranges used in queries so they don't
produce result sets that are too large. Currently there are no limits enforced
on date ranges.
Queries can also be configured on a global level for setting the number of
results per page via a Spring property `pcap.page.size`. By default, this value
is set to 10 pcaps per page, but you may choose to set this value higher
based on observing frequenetly-run query result sizes. This setting works in
conjunction with the property for setting finalizer threadpool size when
optimizing query performance.
[43/51] [abbrv] metron git commit: METRON-1725 Add ability to specify YARN queue for pcap jobs (merrimanr) closes apache/metron#1153
METRON-1725 Add ability to specify YARN queue for pcap jobs (merrimanr) closes apache/metron#1153 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7a8c2467 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7a8c2467 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7a8c2467 Branch: refs/heads/master Commit: 7a8c246748a2c9e8c5b9230800b075dd99a7f3a4 Parents: 73dc63e Author: merrimanr Authored: Fri Aug 10 16:46:31 2018 -0500 Committer: rmerriman Committed: Fri Aug 10 16:46:31 2018 -0500 -- .../CURRENT/configuration/metron-rest-env.xml | 9 ++ .../package/scripts/params/params_linux.py | 1 + .../METRON/CURRENT/package/templates/metron.j2 | 1 + .../METRON/CURRENT/themes/metron_theme.json | 10 ++ metron-interface/metron-rest/README.md | 2 ++ .../src/main/config/rest_application.yml| 1 + .../apache/metron/rest/MetronRestConstants.java | 1 + .../metron/rest/config/PcapJobSupplier.java | 2 +- .../rest/service/impl/PcapServiceImpl.java | 12 ++- .../apache/metron/rest/mock/MockPcapJob.java| 8 + .../rest/service/impl/PcapServiceImplTest.java | 7 - metron-platform/metron-pcap-backend/README.md | 2 ++ .../org/apache/metron/pcap/query/CliParser.java | 4 +++ .../org/apache/metron/pcap/query/PcapCli.java | 3 ++ .../apache/metron/pcap/query/PcapCliTest.java | 33 ++-- .../apache/metron/pcap/config/PcapConfig.java | 10 ++ 16 files changed, 101 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/7a8c2467/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml index 20f9767..895c091 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml @@ -192,5 +192,14 @@ The number of pcaps written to a page/file as a result of a pcap query. 10 + +pcap_yarn_queue +Pcap YARN Queue +The YARN queue pcap jobs will be submitted to. + + +true + + http://git-wip-us.apache.org/repos/asf/metron/blob/7a8c2467/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 73d3469..4f8a9a7 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -390,6 +390,7 @@ pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path'] pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pcap_base_interim_result_path'] pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path'] pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size'] +pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue'] pcap_configured_flag_file = status_params.pcap_configured_flag_file # MapReduce http://git-wip-us.apache.org/repos/asf/metron/blob/7a8c2467/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 index 278d6f8..55422d0 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
[24/51] [abbrv] metron git commit: METRON-1700 Create REST endpoint to get job configuration (merrimanr) closes apache/metron#1135
METRON-1700 Create REST endpoint to get job configuration (merrimanr) closes
apache/metron#1135
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2b6959b4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2b6959b4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2b6959b4
Branch: refs/heads/master
Commit: 2b6959b46db79256d12b8522385212559cabada2
Parents: d5eb56a
Author: merrimanr
Authored: Thu Aug 2 12:17:37 2018 -0500
Committer: rmerriman
Committed: Thu Aug 2 12:17:37 2018 -0500
--
metron-interface/metron-rest/README.md | 10 ++
.../metron/rest/controller/PcapController.java | 16 +++
.../apache/metron/rest/service/PcapService.java | 71 -
.../rest/service/impl/PcapServiceImpl.java | 54 +-
.../PcapControllerIntegrationTest.java | 50 ++
.../apache/metron/rest/mock/MockPcapJob.java| 4 +-
.../rest/service/impl/PcapServiceImplTest.java | 100 +++
.../java/org/apache/metron/job/Statusable.java | 2 +
.../java/org/apache/metron/pcap/mr/PcapJob.java | 5 +
9 files changed, 306 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/2b6959b4/metron-interface/metron-rest/README.md
--
diff --git a/metron-interface/metron-rest/README.md
b/metron-interface/metron-rest/README.md
index 68ec559..489cd9f 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -259,6 +259,8 @@ Request and Response objects are JSON formatted. The JSON
schemas are available
| [ `GET /api/v1/pcap/{jobId}`](#get-apiv1pcapjobid)|
| [ `GET /api/v1/pcap/{jobId}/pdml`](#get-apiv1pcapjobidpdml)|
| [ `GET /api/v1/pcap/{jobId}/raw`](#get-apiv1pcapjobidraw)|
+| [ `DELETE /api/v1/pcap/kill/{jobId}`](#delete-apiv1pcapkilljobid)|
+| [ `GET /api/v1/pcap/{jobId}/config`](#get-apiv1pcapjobidconfig)|
| [ `GET /api/v1/search/search`](#get-apiv1searchsearch)|
| [ `POST /api/v1/search/search`](#post-apiv1searchsearch)|
| [ `POST /api/v1/search/group`](#post-apiv1searchgroup)|
@@ -556,6 +558,14 @@ Request and Response objects are JSON formatted. The JSON
schemas are available
* jobId - Job ID of submitted job
* Returns:
* 200 - Kills passed job.
+
+### `GET /api/v1/pcap/{jobId}/config`
+ * Description: Gets job configuration for Pcap query job.
+ * Input:
+* jobId - Job ID of submitted job
+ * Returns:
+* 200 - Returns a map of job properties for the Job ID.
+* 404 - Job is missing.
### `POST /api/v1/search/search`
* Description: Searches the indexing store. GUIDs must be quoted to ensure
correct results.
http://git-wip-us.apache.org/repos/asf/metron/blob/2b6959b4/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
index 13a623a..97713d8 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
@@ -46,6 +46,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
+import java.util.Map;
@RestController
@RequestMapping("/api/v1/pcap")
@@ -161,4 +162,19 @@ public class PcapController {
}
}
+ @ApiOperation(value = "Gets job configuration for Pcap query job.")
+ @ApiResponses(value = {
+ @ApiResponse(message = "Returns a map of job properties for the Job
ID.", code = 200),
+ @ApiResponse(message = "Job is missing.", code = 404)
+ })
+ @RequestMapping(value = "/{jobId}/config", method = RequestMethod.GET)
+ ResponseEntity> getConfiguration(@ApiParam(name="jobId",
value="Job ID of submitted job", required=true)@PathVariable String jobId)
throws RestException {
+Map configuration =
pcapQueryService.getConfiguration(SecurityUtils.getCurrentUser(), jobId);
+if (configuration != null) {
+ return new ResponseEntity<>(configuration, HttpStatus.OK);
+} else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+}
+ }
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/2b6959b4/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
--
diff --git
a/metron-interface/metron-rest/src/
[36/51] [abbrv] metron git commit: METRON-1723 PCAP UI - Unable to select/copy from packets details in PCAP query panel (sardell via merrimanr) closes apache/metron#1139
METRON-1723 PCAP UI - Unable to select/copy from packets details in PCAP query
panel (sardell via merrimanr) closes apache/metron#1139
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/5b3e2c33
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/5b3e2c33
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/5b3e2c33
Branch: refs/heads/master
Commit: 5b3e2c337b3772bcd7d18cedac75877bbe6f7735
Parents: 52de126
Author: sardell
Authored: Thu Aug 9 07:44:15 2018 -0500
Committer: rmerriman
Committed: Thu Aug 9 07:44:15 2018 -0500
--
.../src/app/pcap/pcap-packet/pcap-packet.component.html | 2 +-
.../src/app/pcap/pcap-packet/pcap-packet.component.scss | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/5b3e2c33/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
index f7d9dbe..dd9afa1 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
@@ -13,7 +13,7 @@
-->
{{ proto.showname
}}
-
+
{{ field.name
}}
{{ field.showname }}
http://git-wip-us.apache.org/repos/asf/metron/blob/5b3e2c33/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
index e9d4662..02ce7b6 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
@@ -27,10 +27,10 @@
}
.proto-fields {
- margin-left: 1em;
+ cursor: text;
+ padding-left: 1em;
}
-
.field-name {
display: none
}
[07/51] [abbrv] metron git commit: METRON-1674 Create REST endpoint for job status abstraction (merrimanr) closes apache/metron#1109
METRON-1674 Create REST endpoint for job status abstraction (merrimanr) closes
apache/metron#1109
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/39ae9f46
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/39ae9f46
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/39ae9f46
Branch: refs/heads/master
Commit: 39ae9f4642073d3d4f0fa423339dd97f85974588
Parents: dbbf624
Author: merrimanr
Authored: Thu Jul 19 11:01:49 2018 -0500
Committer: rmerriman
Committed: Thu Jul 19 11:01:49 2018 -0500
--
.../rest/model/pcap/FixedPcapOptions.java | 42
.../rest/model/pcap/FixedPcapRequest.java | 72 --
.../metron/rest/model/pcap/PcapRequest.java | 65 +++---
.../metron/rest/model/pcap/PcapStatus.java | 91
.../apache/metron/rest/MetronRestConstants.java | 6 +-
.../apache/metron/rest/config/PcapConfig.java | 14 +-
.../metron/rest/config/PcapJobSupplier.java | 54 +
.../metron/rest/controller/PcapController.java | 34 +--
.../apache/metron/rest/service/PcapService.java | 6 +-
.../rest/service/impl/PcapServiceImpl.java | 126 ++-
.../src/main/resources/application.yml | 6 +-
.../apache/metron/rest/config/TestConfig.java | 17 +-
.../PcapControllerIntegrationTest.java | 127 ++-
.../apache/metron/rest/mock/MockPcapJob.java| 106 ++---
.../metron/rest/mock/MockPcapJobSupplier.java | 36 +++
.../rest/service/impl/PcapServiceImplTest.java | 217 +--
.../common/configuration/ConfigOption.java | 12 +-
.../apache/metron/job/JobNotFoundException.java | 30 +++
.../apache/metron/job/RuntimeJobException.java | 30 +++
.../metron/job/manager/InMemoryJobManager.java | 11 +-
.../org/apache/metron/pcap/query/PcapCli.java | 12 -
.../PcapTopologyIntegrationTest.java| 6 +-
.../apache/metron/pcap/query/PcapCliTest.java | 2 -
.../apache/metron/pcap/config/PcapOptions.java | 2 +
.../metron/pcap/finalizer/PcapCliFinalizer.java | 4 +-
.../metron/pcap/finalizer/PcapFinalizer.java| 8 +-
.../pcap/finalizer/PcapRestFinalizer.java | 22 +-
.../java/org/apache/metron/pcap/mr/PcapJob.java | 18 +-
28 files changed, 882 insertions(+), 294 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/39ae9f46/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapOptions.java
--
diff --git
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapOptions.java
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapOptions.java
new file mode 100644
index 000..5e77005
--- /dev/null
+++
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapOptions.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model.pcap;
+
+import org.apache.metron.common.configuration.ConfigOption;
+
+public enum FixedPcapOptions implements ConfigOption {
+ IP_SRC_ADDR("ipSrcAddr"),
+ IP_DST_ADDR("ipDstAddr"),
+ IP_SRC_PORT("ipSrcPort"),
+ IP_DST_PORT("ipDstPort"),
+ PROTOCOL("protocol"),
+ PACKET_FILTER("packetFilter"),
+ INCLUDE_REVERSE("includeReverse")
+ ;
+
+ String key;
+
+ FixedPcapOptions(String key) {
+this.key = key;
+ }
+
+ @Override
+ public String getKey() {
+return key;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/39ae9f46/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
--
diff --git
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/r
[32/51] [abbrv] metron git commit: METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr) closes apache/metron#1122
METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr)
closes apache/metron#1122
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3e778592
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3e778592
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3e778592
Branch: refs/heads/master
Commit: 3e7785920e8f0c5aadf36e05dbcfa69252cc5574
Parents: 037b50b
Author: sardell
Authored: Tue Aug 7 13:58:08 2018 -0500
Committer: rmerriman
Committed: Tue Aug 7 13:58:08 2018 -0500
--
.../metron-alerts/src/app/app.component.html|2 +-
.../metron-alerts/src/app/app.component.ts |3 +
.../src/app/pcap/model/pcap-status-response.ts |1 +
.../src/app/pcap/model/pcap.mock.ts | 39 +
.../src/app/pcap/model/pdml.mock.ts | 62 +
.../metron-alerts/src/app/pcap/model/pdml.ts|6 +-
.../app/pcap/pcap-list/pcap-list.component.html |2 +-
.../pcap/pcap-list/pcap-list.component.spec.ts | 17 +
.../pcap-packet-line.component.spec.ts | 1336 +
.../pcap/pcap-packet/pcap-packet.component.html | 11 +-
.../pcap-packet/pcap-packet.component.spec.ts | 1278 +
.../pcap/pcap-panel/pcap-panel.component.html |6 +-
.../pcap/pcap-panel/pcap-panel.component.scss | 18 -
.../pcap-panel/pcap-panel.component.spec.ts | 262 ++-
.../app/pcap/pcap-panel/pcap-panel.component.ts | 52 +-
.../src/app/pcap/service/pcap.service.spec.ts | 1809 +-
.../src/app/pcap/service/pcap.service.ts|7 +-
.../src/environments/environment.e2e.ts |3 +-
.../src/environments/environment.prod.ts|3 +-
.../src/environments/environment.ts |3 +-
metron-interface/metron-alerts/src/index.html |2 +-
21 files changed, 674 insertions(+), 4248 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/app.component.html
--
diff --git a/metron-interface/metron-alerts/src/app/app.component.html
b/metron-interface/metron-alerts/src/app/app.component.html
index 1b992ca..c3a5d3c 100644
--- a/metron-interface/metron-alerts/src/app/app.component.html
+++ b/metron-interface/metron-alerts/src/app/app.component.html
@@ -11,7 +11,7 @@
OR CONDITIONS OF ANY KIND, either express or implied. See the License
for
the specific language governing permissions and limitations under the
License.
-->
-
+
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/app.component.ts
--
diff --git a/metron-interface/metron-alerts/src/app/app.component.ts
b/metron-interface/metron-alerts/src/app/app.component.ts
index 2a3c15e..52dee23 100644
--- a/metron-interface/metron-alerts/src/app/app.component.ts
+++ b/metron-interface/metron-alerts/src/app/app.component.ts
@@ -17,6 +17,7 @@
*/
import { Component, OnInit } from '@angular/core';
import {AuthenticationService} from './service/authentication.service';
+import { environment } from 'environments/environment';
declare var $;
@@ -27,11 +28,13 @@ declare var $;
})
export class AppComponent implements OnInit {
loggedIn = false;
+ noTransition = false;
constructor(private authService: AuthenticationService) {
this.authService.onLoginEvent.subscribe(result => {
this.loggedIn = result;
});
+this.noTransition = environment.noTransition;
}
ngOnInit(): void {
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
index d4d9a5e..09c4985 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
@@ -18,6 +18,7 @@
export class PcapStatusResponse {
jobId: string;
jobStatus: string;
+ description: string;
percentComplete: number;
pageTotal: number;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
new file mode 100644
index 000..c867fe9
--- /dev/null
+++ b/metron-interface/metron-alerts/s
[09/51] [abbrv] metron git commit: METRON-1638 Retrieve Pcap results in pdml format (merrimanr) closes apache/metron#1120
METRON-1638 Retrieve Pcap results in pdml format (merrimanr) closes
apache/metron#1120
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3e5ef41d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3e5ef41d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3e5ef41d
Branch: refs/heads/master
Commit: 3e5ef41d9b8639fb1155686e615c02a59b735397
Parents: f316d15
Author: merrimanr
Authored: Thu Jul 19 17:13:22 2018 -0500
Committer: rmerriman
Committed: Thu Jul 19 17:13:22 2018 -0500
--
dependencies_with_url.csv | 3 +
.../docker/rpm-docker/SPECS/metron.spec | 1 +
metron-interface/metron-rest-client/pom.xml | 5 +
.../apache/metron/rest/model/pcap/Field.java| 164
.../apache/metron/rest/model/pcap/Packet.java | 53
.../org/apache/metron/rest/model/pcap/Pdml.java | 103
.../apache/metron/rest/model/pcap/Proto.java| 114 +
metron-interface/metron-rest/README.md | 34 ++-
metron-interface/metron-rest/pom.xml| 1 -
.../src/main/config/rest_application.yml| 5 +-
.../apache/metron/rest/MetronRestConstants.java | 1 +
.../apache/metron/rest/config/PcapConfig.java | 6 +
.../metron/rest/controller/PcapController.java | 31 ++-
.../apache/metron/rest/service/PcapService.java | 6 +
.../rest/service/impl/PcapServiceImpl.java | 47 +++-
.../service/impl/PcapToPdmlScriptWrapper.java | 59 +
.../src/main/scripts/pcap_to_pdml.sh| 19 ++
.../apache/metron/rest/config/TestConfig.java | 7 +
.../PcapControllerIntegrationTest.java | 65 +++--
.../rest/mock/MockPcapToPdmlScriptWrapper.java | 55
.../rest/service/impl/PcapServiceImplTest.java | 250 +--
metron-interface/pom.xml| 3 +
22 files changed, 981 insertions(+), 51 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/3e5ef41d/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 40d3e06..bf3e382 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -141,12 +141,15 @@
com.fasterxml.jackson.dataformat:jackson-dataformat-smile:jar:2.6.6:compile,ASLv
com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:2.6.6:compile,ASLv2,https://github.com/FasterXML/jackson
com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:jar:2.7.4:compile,ASLv2,http://wiki.fasterxml.com/JacksonForCbor
com.fasterxml.jackson.dataformat:jackson-dataformat-smile:jar:2.7.4:compile,ASLv2,http://wiki.fasterxml.com/JacksonForSmile
+com.fasterxml.jackson.dataformat:jackson-dataformat-xml:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson
com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:2.7.4:compile,ASLv2,https://github.com/FasterXML/jackson
com.fasterxml.jackson.datatype:jackson-datatype-joda:jar:2.8.1:compile,ASLv2,https://github.com/FasterXML/jackson-datatype-joda
com.fasterxml.jackson.datatype:jackson-datatype-joda:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-datatype-joda
com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-modules-java8
com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-modules-java8
com.fasterxml.jackson.module:jackson-module-parameter-names:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-modules-java8
+com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-modules-java8
+com.fasterxml.woodstox:woodstox-core:jar:5.0.3:compile,ASLv2,https://github.com/FasterXML/woodstox
com.fasterxml:classmate:jar:1.3.1:compile,ASLv2,http://github.com/cowtowncoder/java-classmate
com.fasterxml:classmate:jar:1.3.4:compile,ASLv2,http://github.com/cowtowncoder/java-classmate
com.google.code.gson:gson:jar:2.2.4:compile,The Apache Software License,
Version 2.0,http://code.google.com/p/google-gson/
http://git-wip-us.apache.org/repos/asf/metron/blob/3e5ef41d/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 4b88fd0..3f090c8 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -419,6 +419,7 @@ This package installs the Metron Rest %{metron_home}
%dir %{metron_home}/lib
%{metron_home}/config/rest_application.yml
%{metron_home
[11/51] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1554-pcap-query-panel
Merge branch 'master' into feature/METRON-1554-pcap-query-panel Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b580a69f Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b580a69f Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b580a69f Branch: refs/heads/master Commit: b580a69f1d9d3346171d749dcd6f791a9bc6d894 Parents: a5a5139 aceca46 Author: Michael Miklavcic Authored: Fri Jul 20 09:40:09 2018 -0600 Committer: Michael Miklavcic Committed: Fri Jul 20 09:40:09 2018 -0600 -- .travis.yml | 14 +- .../metron/profiler/client/window/Window.java | 8 +- .../profiler/client/window/WindowProcessor.java | 35 +- .../client/window/predicates/DayPredicates.java | 2 +- .../profiler/DefaultMessageDistributor.java | 18 +- .../ansible/playbooks/docker_probe_install.yml | 2 +- .../ansible/roles/bro/vars/main.yml | 2 +- .../centos6/ansible/inventory/group_vars/all| 2 +- .../ubuntu14/ansible/inventory/group_vars/all | 2 +- .../package/scripts/alerts_ui_commands.py | 9 +- .../package/scripts/management_ui_commands.py | 9 +- metron-deployment/scripts/platform-info.sh | 4 + .../alert-details-status.e2e-spec.ts| 129 +++--- .../meta-alerts/meta-alert.e2e-spec.ts | 30 +- metron-interface/metron-alerts/karma.conf.js| 15 +- metron-interface/metron-alerts/package.json | 3 +- metron-interface/metron-alerts/pom.xml | 14 +- .../apache/metron/rest/config/KafkaConfig.java | 4 +- .../service/impl/StormAdminServiceImpl.java | 23 +- .../rest/service/impl/StormCLIWrapper.java | 25 +- .../common/bolt/ConfiguredParserBolt.java | 10 +- .../configuration/IndexingConfigurations.java | 2 +- .../configuration/SensorParserConfig.java | 12 +- .../writer/ConfigurationsStrategies.java| 2 +- .../apache/metron/common/error/MetronError.java | 64 +-- .../common/performance/PerformanceLogger.java | 10 +- .../apache/metron/common/system/FakeClock.java | 18 +- .../metron/common/utils/ReflectionUtils.java| 2 +- .../configurations/ConfigurationsUpdater.java | 1 - .../common/bolt/ConfiguredParserBoltTest.java | 2 +- .../metron/common/error/MetronErrorTest.java| 16 +- .../indexing/dao/metaalert/MetaAlertDao.java| 6 +- .../dao/metaalert/MetaAlertIntegrationTest.java | 56 +++ .../metron/management/KafkaFunctions.java | 109 + .../KafkaFunctionsIntegrationTest.java | 92 + .../metron-parsers/3rdPartyParser.md| 408 +-- metron-platform/metron-parsers/README.md| 25 +- .../apache/metron/parsers/bolt/ParserBolt.java | 241 ++- .../apache/metron/parsers/bolt/WriterBolt.java | 3 +- .../parsers/topology/ParserComponents.java | 67 +++ .../parsers/topology/ParserTopologyBuilder.java | 200 ++--- .../parsers/topology/ParserTopologyCLI.java | 264 .../parsers/topology/config/ValueSupplier.java | 3 +- .../metron/parsers/bolt/ParserBoltTest.java | 182 ++--- .../metron/parsers/bolt/WriterBoltTest.java | 30 +- .../parsers/integration/ParserDriver.java | 50 ++- .../components/ParserTopologyComponent.java | 58 +-- .../parsers/topology/ParserTopologyCLITest.java | 318 ++- ...pleHbaseEnrichmentWriterIntegrationTest.java | 3 +- .../integration/WriterBoltIntegrationTest.java | 150 ++- .../src/main/config/schema/metaalert/schema.xml | 6 +- .../metron/writer/BulkWriterComponent.java | 27 +- .../metron/writer/BulkWriterComponentTest.java | 34 +- .../stellar/common/CachingStellarProcessor.java | 2 +- .../org/apache/metron/stellar/dsl/Context.java | 2 +- .../dsl/functions/DateFunctionsTest.java| 3 +- use-cases/forensic_clustering/README.md | 228 +++ use-cases/parser_chaining/README.md | 14 + .../aggregated_parser_chaining_flow.svg | 14 + .../aggregated_parser_chaining_flow.xml | 14 + use-cases/typosquat_detection/README.md | 9 +- 61 files changed, 2109 insertions(+), 998 deletions(-) --
[18/51] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1554-pcap-query-panel
Merge branch 'master' into feature/METRON-1554-pcap-query-panel
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7dff4def
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7dff4def
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7dff4def
Branch: refs/heads/master
Commit: 7dff4def23a7c839ea599a120ab6d4cf5b9bbbef
Parents: 6c90724 ccdbeff
Author: cstella
Authored: Thu Jul 26 12:29:04 2018 -0400
Committer: cstella
Committed: Thu Jul 26 12:29:04 2018 -0400
--
.../docker/rpm-docker/SPECS/metron.spec | 1 +
.../e2e/utils/clean_metron_update_table.ts | 18 ++
.../metron-alerts/e2e/utils/e2e_util.ts | 20 +-
.../alert-filters.component.spec.ts | 18 ++
.../alert-filters/alert-filters.component.ts| 20 +-
.../table-view/table-view.component.spec.ts | 18 ++
.../tree-view/tree-view.component.spec.ts | 18 ++
.../alerts/meta-alerts/meta-alerts.module.ts| 18 ++
.../date-picker/date-picker.component.spec.ts | 18 ++
.../shared/date-picker/date-picker.component.ts | 18 ++
.../shared/date-picker/date-picker.module.ts| 18 ++
.../alert-severity-hexagon.directive.spec.ts| 18 ++
.../app/shared/group-by/group-by.component.ts | 18 ++
.../metron-alerts/src/app/shared/index.ts | 18 ++
.../src/app/shared/pipes/map-keys.pipe.spec.ts | 18 ++
.../src/app/shared/pipes/map-keys.pipe.ts | 18 ++
.../src/app/shared/pipes/time-lapse.pipe.ts | 18 ++
.../time-range/time-range.component.spec.ts | 18 ++
.../app/shared/time-range/time-range.module.ts | 18 ++
metron-interface/metron-alerts/src/polyfills.ts | 18 ++
.../metron-config/e2e/utils/e2e_util.ts | 18 ++
.../src/app/app.config.interface.ts | 18 ++
.../metron-config/src/app/app.config.ts | 18 ++
.../sensor-grok/sensor-grok.component.ts| 18 ++
.../sensor-parser-config-readonly/index.ts | 18 ++
.../app/sensors/sensor-parser-config/index.ts | 18 ++
.../shared/ace-editor/ace-editor.component.ts | 18 ++
.../app/shared/ace-editor/ace-editor.module.ts | 18 ++
.../advanced-config-form.module.ts | 18 ++
.../src/app/shared/metron-modal/index.ts| 18 ++
.../shared/metron-table/metron-sorter/index.ts | 18 ++
.../metron-sorter.component.spec.ts | 18 ++
.../metron-sorter/metron-sorter.component.ts| 18 ++
.../shared/metron-table/metron-table.module.ts | 18 ++
.../src/app/shared/multiple-input/index.ts | 18 ++
.../multiple-input/multiple-input.module.ts | 18 ++
.../src/app/shared/number-spinner/index.ts | 18 ++
.../number-spinner/number-spinner.module.ts | 18 ++
.../shared/sample-data/sample-data.module.ts| 18 ++
.../src/environments/environment.prod.ts| 18 ++
.../src/environments/environment.ts | 18 ++
metron-interface/metron-rest/README.md | 12 +-
.../parsed/jsonMapExampleParsed | 6 +
.../raw/jsonMapExampleOutput| 3 +
metron-platform/metron-parsers/README.md| 5 +
.../zookeeper/parsers/jsonMapWrappedQuery.json | 5 +
.../metron/parsers/json/JSONMapParser.java | 40 +++-
.../JSONMapWrappedQueryIntegrationTest.java | 37
.../integration/ParserIntegrationTest.java | 22 +-
.../json/JSONMapParserWrappedQueryTest.java | 199 +++
pom.xml | 19 +-
51 files changed, 1044 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/7dff4def/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--
http://git-wip-us.apache.org/repos/asf/metron/blob/7dff4def/metron-interface/metron-rest/README.md
--
diff --cc metron-interface/metron-rest/README.md
index 2d9a535,caa9c78..68ec559
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@@ -253,15 -242,8 +253,15 @@@ Request and Response objects are JSON f
| [ `GET /api/v1/metaalert/add/alert`](#get-apiv1metaalertaddalert)|
| [ `GET /api/v1/metaalert/remove/alert`](#get-apiv1metaalertremovealert)|
| [ `GET
/api/v1/metaalert/update/status/{guid}/{status}`](#get-apiv1metaalertupdatestatusguidstatus)|
+| [ `POST /api/v1/pcap/fixed`](#post-apiv1pcapfixed)|
+| [ `POST /api/v1/pcap/query`](#post-apiv1pcapquery)|
+| [ `GET /api/v1/pcap`](#get-apiv1pcap)|
+| [ `GET /api/v1/pcap/{jobId}`](#get-apiv1pcapjobid)|
+| [ `GET /api/v1/pcap/{jobId}/pdml`](#get-apiv1pcapjobidpdml)|
+| [ `GET /api/v1/pcap/{jobId}/raw`](#get-apiv1pcapjobidraw)|
+| [ `GET /api/v1/search/search`](#get-apiv1searchsearch)|
- | [ `POST
[51/51] [abbrv] metron git commit: METRON-1554 Pcap Query Panel (merrimanr) closes apache/metron#1169
METRON-1554 Pcap Query Panel (merrimanr) closes apache/metron#1169 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9fdccba3 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9fdccba3 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9fdccba3 Branch: refs/heads/master Commit: 9fdccba371c1f1e0dcb79e00a7207a934b79b64c Parents: 308c2b2 14e80b3 Author: merrimanr Authored: Fri Aug 17 10:26:51 2018 -0500 Committer: merrimanr Committed: Fri Aug 17 10:28:09 2018 -0500 -- dependencies_with_url.csv |4 + metron-deployment/amazon-ec2/conf/defaults.yml |1 - metron-deployment/amazon-ec2/playbook.yml |2 +- .../ansible/playbooks/metron_install.yml|7 - .../roles/ambari_config/vars/single_node_vm.yml |1 + .../roles/metron_pcapservice/defaults/main.yml | 28 - .../roles/metron_pcapservice/meta/main.yml | 19 - .../metron_pcapservice/tasks/config-hbase.yml | 26 - .../roles/metron_pcapservice/tasks/main.yml | 25 - .../metron_pcapservice/tasks/pcapservice.yml| 30 - .../templates/metron-pcapservice-logrotate.yml | 27 - .../metron_pcapservice/templates/pcapservice| 84 - .../ubuntu14/ansible/inventory/hosts|4 - .../CURRENT/configuration/metron-rest-env.xml | 48 + .../package/scripts/params/params_linux.py | 16 + .../package/scripts/params/status_params.py |7 + .../CURRENT/package/scripts/rest_commands.py| 62 + .../CURRENT/package/scripts/rest_master.py |9 + .../METRON/CURRENT/package/templates/metron.j2 |6 + .../METRON/CURRENT/themes/metron_theme.json | 30 + .../docker/rpm-docker/SPECS/metron.spec |1 + .../metron-alerts/package-lock.json | 3719 +- .../metron-alerts/src/app/app-routing.module.ts |3 +- .../metron-alerts/src/app/app.component.html| 10 +- .../metron-alerts/src/app/app.component.scss| 18 + .../metron-alerts/src/app/app.component.ts |3 + .../metron-alerts/src/app/app.module.ts |6 +- .../src/app/pcap/model/pcap-pagination.ts | 21 + .../src/app/pcap/model/pcap-status-response.ts | 24 + .../src/app/pcap/model/pcap.mock.ts | 39 + .../src/app/pcap/model/pcap.request.ts | 29 + .../src/app/pcap/model/pdml.mock.ts | 93 + .../metron-alerts/src/app/pcap/model/pdml.ts| 50 + .../pcap-filters/pcap-filters.component.html| 62 + .../pcap-filters/pcap-filters.component.scss| 69 + .../pcap-filters/pcap-filters.component.spec.ts | 463 +++ .../pcap/pcap-filters/pcap-filters.component.ts | 77 + .../app/pcap/pcap-list/pcap-list.component.html | 38 + .../app/pcap/pcap-list/pcap-list.component.scss | 21 + .../pcap/pcap-list/pcap-list.component.spec.ts | 98 + .../app/pcap/pcap-list/pcap-list.component.ts | 43 + .../pcap-packet-line.component.html | 19 + .../pcap-packet-line.component.scss | 17 + .../pcap-packet-line.component.spec.ts | 180 + .../pcap-packet-line.component.ts | 56 + .../pcap/pcap-packet/pcap-packet.component.html | 22 + .../pcap/pcap-packet/pcap-packet.component.scss | 36 + .../pcap-packet/pcap-packet.component.spec.ts | 79 + .../pcap/pcap-packet/pcap-packet.component.ts | 34 + .../pcap-pagination.component.html | 19 + .../pcap-pagination.component.scss | 40 + .../pcap-pagination.component.spec.ts | 78 + .../pcap-pagination.component.ts| 41 + .../pcap/pcap-panel/pcap-panel.component.html | 31 + .../pcap/pcap-panel/pcap-panel.component.scss | 67 + .../pcap-panel/pcap-panel.component.spec.ts | 557 +++ .../app/pcap/pcap-panel/pcap-panel.component.ts | 164 + .../metron-alerts/src/app/pcap/pcap.module.ts | 54 + .../metron-alerts/src/app/pcap/pcap.routing.ts | 27 + .../src/app/pcap/service/pcap.service.spec.ts | 197 + .../src/app/pcap/service/pcap.service.ts| 88 + .../shared/directives/alert-search.directive.ts |2 +- .../src/environments/environment.e2e.ts |3 +- .../src/environments/environment.prod.ts|3 +- .../src/environments/environment.ts |3 +- metron-interface/metron-alerts/src/index.html |2 +- metron-interface/metron-rest-client/pom.xml | 10 + .../apache/metron/rest/model/PcapResponse.java | 38 + .../apache/metron/rest/model/pcap/Field.java| 154 + .../rest/model/pcap/FixedPcapOptions.java | 42 + .../rest/model/pcap/FixedPcapRequest.java | 116 + .../apache/metron/rest/model/pcap/Packet.java | 53 + .../metron/rest/model/pcap/PcapRequest.java | 83 + .../metron/rest/model/pcap/PcapStatus.java | 87 + .../org/apache/metron/rest/model/pcap/Pdml.java
[41/51] [abbrv] metron git commit: METRON-1702 Reload a running job in the UI (merrimanr) closes apache/metron#1156
METRON-1702 Reload a running job in the UI (merrimanr) closes apache/metron#1156
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/05316a4d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/05316a4d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/05316a4d
Branch: refs/heads/master
Commit: 05316a4d989d7546058e73b3a75eef435f2608a0
Parents: b29833c
Author: merrimanr
Authored: Fri Aug 10 08:27:06 2018 -0500
Committer: rmerriman
Committed: Fri Aug 10 08:27:06 2018 -0500
--
.../src/app/pcap/model/pcap.request.ts | 4 +-
.../pcap-filters/pcap-filters.component.spec.ts | 60 +-
.../pcap/pcap-filters/pcap-filters.component.ts | 26 --
.../pcap/pcap-panel/pcap-panel.component.html | 2 +-
.../pcap-panel/pcap-panel.component.spec.ts | 37 +
.../app/pcap/pcap-panel/pcap-panel.component.ts | 87 +---
.../src/app/pcap/service/pcap.service.spec.ts | 44 ++
.../src/app/pcap/service/pcap.service.ts| 15
.../rest/service/impl/PcapServiceImpl.java | 10 +--
.../rest/service/impl/PcapServiceImplTest.java | 36
10 files changed, 254 insertions(+), 67 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/05316a4d/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
index d00a6ac..8afc963 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
@@ -20,9 +20,9 @@ export class PcapRequest {
startTimeMs: number = 0;
endTimeMs: number = 15;
ipSrcAddr: string = '';
- ipSrcPort: number = 0;
+ ipSrcPort: number;
ipDstAddr: string = '';
- ipDstPort: number = 0;
+ ipDstPort: number;
protocol: string = '';
packetFilter: string = '';
includeReverse: boolean = false;
http://git-wip-us.apache.org/repos/asf/metron/blob/05316a4d/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
index 4336b22..c1d8664 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
@@ -21,7 +21,7 @@ import { By } from '@angular/platform-browser';
import { PcapFiltersComponent } from './pcap-filters.component';
import { FormsModule } from '../../../../node_modules/@angular/forms';
-import { Component, Input, Output, EventEmitter, DebugElement } from
'@angular/core';
+import { Component, Input, Output, EventEmitter, DebugElement, SimpleChange }
from '@angular/core';
import { PcapRequest } from '../model/pcap.request';
@Component({
@@ -171,6 +171,14 @@ describe('PcapFiltersComponent', () => {
component.onSubmit();
});
+ it('Port fields should be missing by default', () => {
+component.search.emit = (model: PcapRequest) => {
+ expect(model.ipSrcPort).toBeFalsy();
+ expect(model.ipDstPort).toBeFalsy();
+};
+component.onSubmit();
+ });
+
it('Filter should have an output called search', () => {
component.search.subscribe((filterModel) => {
expect(filterModel).toBeDefined();
@@ -194,14 +202,60 @@ describe('PcapFiltersComponent', () => {
expect(fixture.componentInstance.model.hasOwnProperty('startTimeMs')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('endTimeMs')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('ipSrcAddr')).toBeTruthy();
-
expect(fixture.componentInstance.model.hasOwnProperty('ipSrcPort')).toBeTruthy();
+
expect(fixture.componentInstance.model.hasOwnProperty('ipSrcPort')).toBeFalsy();
expect(fixture.componentInstance.model.hasOwnProperty('ipDstAddr')).toBeTruthy();
-
expect(fixture.componentInstance.model.hasOwnProperty('ipDstPort')).toBeTruthy();
+
expect(fixture.componentInstance.model.hasOwnProperty('ipDstPort')).toBeFalsy();
expect(fixture.componentInstance.model.hasOwnProperty('protocol')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('packetFilter')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('includeReverse')).toBeTruthy();
});
+ it('should update request on changes', () => {
+
+let start
[16/51] [abbrv] metron git commit: METRON-1693: Fix Pcap CLI local FS finalizer (mmiklavc via mmiklavc) closes apache/metron#1130
METRON-1693: Fix Pcap CLI local FS finalizer (mmiklavc via mmiklavc) closes
apache/metron#1130
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f1f5dda4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f1f5dda4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f1f5dda4
Branch: refs/heads/master
Commit: f1f5dda4c643394c12ab9f4501c5b806adb21129
Parents: 9d4842f
Author: mmiklavc
Authored: Wed Jul 25 08:26:45 2018 -0600
Committer: Michael Miklavcic
Committed: Wed Jul 25 08:26:45 2018 -0600
--
.../java/org/apache/metron/pcap/query/PcapCli.java | 1 -
.../metron/pcap/finalizer/PcapCliFinalizer.java | 15 ---
.../apache/metron/pcap/finalizer/PcapFinalizer.java | 4 +++-
.../metron/pcap/finalizer/PcapRestFinalizer.java| 10 ++
.../metron/pcap/writer/PcapResultsWriter.java | 16
5 files changed, 41 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
index 1a23740..0b06b0c 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
@@ -87,7 +87,6 @@ public class PcapCli {
try {
config = fixedParser.parse(otherArgs);
commonConfig = config;
-PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path("file:///" +
execDir));
} catch (ParseException | java.text.ParseException e) {
System.err.println(e.getMessage());
System.err.flush();
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
index c379515..c912e58 100644
---
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
+++
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
@@ -18,9 +18,13 @@
package org.apache.metron.pcap.finalizer;
+import java.io.IOException;
+import java.util.List;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.metron.pcap.config.PcapOptions;
+import org.apache.metron.pcap.writer.PcapResultsWriter;
/**
* Write to local FS.
@@ -33,13 +37,18 @@ public class PcapCliFinalizer extends PcapFinalizer {
* as a formatted timestamp + uuid. A final sample format will look as
follows:
*
/base/output/path/pcap-data-201807181911-09855b4ae3204dee8b63760d65198da3+0001.pcap
*/
- private static final String PCAP_CLI_FILENAME_FORMAT =
"%s/pcap-data-%s+%04d.pcap";
+ private static final String PCAP_CLI_FILENAME_FORMAT =
"pcap-data-%s+%04d.pcap";
+
+ @Override
+ protected void write(PcapResultsWriter resultsWriter, Configuration
hadoopConfig,
+ List data, Path outputPath) throws IOException {
+resultsWriter.writeLocal(data, outputPath.toString());
+ }
@Override
protected Path getOutputPath(Map config, int partition) {
-Path finalOutputPath = PcapOptions.FINAL_OUTPUT_PATH.get(config,
PcapOptions.STRING_TO_PATH, Path.class);
String prefix = PcapOptions.FINAL_FILENAME_PREFIX.get(config,
String.class);
-return new Path(String.format(PCAP_CLI_FILENAME_FORMAT, finalOutputPath,
prefix, partition));
+return new Path(String.format(PCAP_CLI_FILENAME_FORMAT, prefix,
partition));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
index 2c55e15..8dcc401 100644
---
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
+++
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
@@ -81,7 +81,7 @@ public abstract class
[46/51] [abbrv] metron git commit: METRON-1735 Empty print status option causes NPE (merrimanr) closes apache/metron#1160
METRON-1735 Empty print status option causes NPE (merrimanr) closes
apache/metron#1160
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8a926dd5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8a926dd5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8a926dd5
Branch: refs/heads/master
Commit: 8a926dd5a4501df8c7ef51b526ad5e4d752a5e19
Parents: bce9b90
Author: merrimanr
Authored: Tue Aug 14 12:59:38 2018 -0500
Committer: rmerriman
Committed: Tue Aug 14 12:59:38 2018 -0500
--
.../org/apache/metron/pcap/query/CliParser.java | 4
.../apache/metron/pcap/query/PcapCliTest.java| 7 ++-
.../apache/metron/pcap/config/PcapConfig.java| 2 +-
.../java/org/apache/metron/pcap/mr/PcapJob.java | 18 +-
.../org/apache/metron/pcap/mr/PcapJobTest.java | 19 +++
5 files changed, 35 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index 5040f90..b9a2a50 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -55,7 +55,6 @@ public class CliParser {
options.addOption(newOption("rpf", "records_per_file", true,
String.format("Number of records to include in each output pcap file (defaults
to %s)", NUM_RECORDS_PER_FILE_DEFAULT)));
options.addOption(newOption("et", "end_time", true, "Packet end time
range. Default is current system time."));
options.addOption(newOption("df", "date_format", true, "Date format to use
for parsing start_time and end_time. Default is to use time in millis since the
epoch."));
-options.addOption(newOption("ps", "print_status", false, "Print the status
of the job as it runs"));
options.addOption(newOption("yq", "yarn_queue", true, "Yarn queue this job
will be submitted to"));
return options;
}
@@ -127,9 +126,6 @@ public class CliParser {
//no-op
}
}
-if (commandLine.hasOption("print_status")) {
- config.setPrintJobStatus(true);
-}
if (commandLine.hasOption("yarn_queue")) {
config.setYarnQueue(commandLine.getOptionValue("yarn_queue"));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index a71e997..3e7aad2 100644
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -172,8 +172,7 @@ public class PcapCliTest {
"-protocol", "6",
"-include_reverse",
"-num_reducers", "10",
-"-records_per_file", "1000",
-"-ps"
+"-records_per_file", "1000"
};
Map query = new HashMap() {{
put(Constants.Fields.SRC_ADDR.getName(), "192.168.1.1");
@@ -217,7 +216,6 @@ public class PcapCliTest {
"-include_reverse",
"-num_reducers", "10",
"-records_per_file", "1000",
-"-ps",
"-yq", "pcap"
};
Map query = new HashMap() {{
@@ -295,8 +293,7 @@ public class PcapCliTest {
"-base_path", "/base/path",
"-base_output_path", "/base/output/path",
"-query", "some query string",
-"-records_per_file", "1000",
-"-ps"
+"-records_per_file", "1000"
};
String query = "some query string";
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-pla
[47/51] [abbrv] metron git commit: METRON-1732: Fix job status liveness bug and parallelize finalizer file writing (mmiklavc via mmiklavc) closes apache/metron#1157
METRON-1732: Fix job status liveness bug and parallelize finalizer file writing (mmiklavc via mmiklavc) closes apache/metron#1157 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d9e1f381 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d9e1f381 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d9e1f381 Branch: refs/heads/master Commit: d9e1f381cc1ae9edfbffc7216abffdbd8f942c09 Parents: 8a926dd Author: mmiklavc Authored: Wed Aug 15 11:30:10 2018 -0600 Committer: Michael Miklavcic Committed: Wed Aug 15 11:30:10 2018 -0600 -- .../CURRENT/configuration/metron-rest-env.xml | 14 +++ .../package/scripts/params/params_linux.py | 1 + .../METRON/CURRENT/package/templates/metron.j2 | 1 + .../METRON/CURRENT/themes/metron_theme.json | 10 +++ metron-interface/metron-rest/README.md | 10 +++ .../src/main/config/rest_application.yml| 1 + .../apache/metron/rest/MetronRestConstants.java | 1 + .../rest/service/impl/PcapServiceImpl.java | 25 +++--- .../apache/metron/rest/mock/MockPcapJob.java| 18 ++-- .../rest/service/impl/PcapServiceImplTest.java | 49 ++- .../org/apache/metron/pcap/query/CliParser.java | 10 ++- .../PcapTopologyIntegrationTest.java| 1 + .../apache/metron/pcap/query/PcapCliTest.java | 21 ++--- .../java/org/apache/metron/pcap/PcapPages.java | 2 +- .../apache/metron/pcap/config/PcapConfig.java | 6 ++ .../metron/pcap/config/PcapGlobalDefaults.java | 1 + .../apache/metron/pcap/config/PcapOptions.java | 3 +- .../metron/pcap/finalizer/PcapFinalizer.java| 70 +-- .../java/org/apache/metron/pcap/mr/PcapJob.java | 90 ++-- .../org/apache/metron/pcap/mr/PcapJobTest.java | 16 +++- 20 files changed, 258 insertions(+), 92 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/d9e1f381/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml index 895c091..767afa3 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml @@ -201,5 +201,19 @@ true + +pcap_finalizer_threadpool_size +Pcap Finalizer Threadpool Size +The number of threads to use when finalizing Pcap jobs. This affects parallelism + around writing out paged files to their final location. + If it's a string and ends with "C", then strip the C and treat it as an integral multiple of + the number of cores. If it's a string and does not end with a C, then treat it as a number in + string form. + +1 + +false + + http://git-wip-us.apache.org/repos/asf/metron/blob/d9e1f381/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 4f8a9a7..115a54c 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -391,6 +391,7 @@ pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pca pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path'] pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size'] pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue'] +pcap_finalizer_threadpool_size= config['configurations']['metron-rest-env']['pcap_finalizer_threadpool_size'] pcap_configured_flag_file = status_params.pcap_configured_flag_file # MapReduce
[29/51] [abbrv] metron git commit: METRON-1675 PCAP UI - Introduce the paging capability (sardell via merrimanr) closes apache/metron#1121
METRON-1675 PCAP UI - Introduce the paging capability (sardell via merrimanr)
closes apache/metron#1121
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/037b50b9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/037b50b9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/037b50b9
Branch: refs/heads/master
Commit: 037b50b9a829d7292db1148fc628d9231da499e1
Parents: 756ceed
Author: sardell
Authored: Tue Aug 7 09:04:33 2018 -0500
Committer: rmerriman
Committed: Tue Aug 7 09:04:33 2018 -0500
--
.../src/app/pcap/model/pcap-pagination.ts | 21 ++
.../src/app/pcap/model/pcap-status-response.ts | 23 ++
.../src/app/pcap/model/pcap.request.ts | 2 +-
.../app/pcap/pcap-list/pcap-list.component.html | 1 +
.../pcap/pcap-list/pcap-list.component.spec.ts | 11 +++
.../app/pcap/pcap-list/pcap-list.component.ts | 18 +++--
.../pcap-pagination.component.html | 19 +
.../pcap-pagination.component.scss | 40 ++
.../pcap-pagination.component.spec.ts | 78
.../pcap-pagination.component.ts| 41 ++
.../pcap/pcap-panel/pcap-panel.component.html | 7 +-
.../pcap-panel/pcap-panel.component.spec.ts | 2 +
.../app/pcap/pcap-panel/pcap-panel.component.ts | 27 +--
.../metron-alerts/src/app/pcap/pcap.module.ts | 10 +--
.../metron-alerts/src/app/pcap/pcap.routing.ts | 2 +-
.../src/app/pcap/service/pcap.service.spec.ts | 2 +-
.../src/app/pcap/service/pcap.service.ts| 10 +--
17 files changed, 279 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/037b50b9/metron-interface/metron-alerts/src/app/pcap/model/pcap-pagination.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/model/pcap-pagination.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap-pagination.ts
new file mode 100644
index 000..ae7619b
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap-pagination.ts
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export class PcapPagination {
+ selectedPage = 1;
+ total: number;
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/037b50b9/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
new file mode 100644
index 000..d4d9a5e
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export class PcapStatusResponse {
+ jobId: string;
+ jobStatus: string;
+ percentComplete: number;
+ pageTotal: number;
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/037b50b9/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.
[30/51] [abbrv] metron git commit: METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr) closes apache/metron#1122
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
index fdaa569..c8cb937 100644
--- a/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
@@ -15,1738 +15,139 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { TestBed, async, inject } from '@angular/core/testing';
+import {
+ TestBed,
+ inject,
+ fakeAsync,
+ tick,
+ discardPeriodicTasks
+} from '@angular/core/testing';
import {
BaseRequestOptions,
HttpModule,
- Http,
Response,
- ResponseOptions
+ ResponseOptions,
+ XHRBackend
} from '@angular/http';
import { MockBackend } from '@angular/http/testing';
+import { Observable } from 'rxjs/Rx';
import { PcapService } from './pcap.service';
+import { PcapStatusResponse } from '../model/pcap-status-response';
import { PcapRequest } from '../model/pcap.request';
+import { fakePdml, fakePacket } from '../model/pdml.mock';
+import { fakePcapStatusResponse, fakePcapRequest } from '../model/pcap.mock';
+
+const jobId = 'job_1234567890123_4567';
+let pdmlJsonMock = fakePdml;
+pdmlJsonMock['pdml']['packet'].push(fakePacket);
describe('PcapService', () => {
beforeEach(() => {
-
TestBed.configureTestingModule({
imports: [HttpModule],
providers: [
PcapService,
-{
- provide: Http,
- useFactory: (mockBackend, options) => {
-return new Http(mockBackend, options);
- },
- deps: [MockBackend, BaseRequestOptions]
-},
+{ provide: XHRBackend, useClass: MockBackend },
MockBackend,
BaseRequestOptions
]
});
});
describe('getPackets()', () => {
-it('should return an Observable',
- inject([PcapService, MockBackend], (pcapService, mockBackend) => {
-
-let request: PcapRequest = {
- startTimeMs: 0,
- endTimeMs: 0,
- ipSrcAddr: '0.0.0.0',
- ipSrcPort: '80',
- ipDstAddr: '0.0.0.0',
- ipDstPort: '80',
- protocol: '*',
- packetFilter: '*',
- includeReverse: false,
-};
-
-mockBackend.connections.subscribe((connection) => {
- connection.mockRespond(new Response(new ResponseOptions({body:
pdml_json()})));
-});
+it('should return an Observable', inject(
+ [PcapService, XHRBackend],
+ (pcapService, mockBackend) => {
let packets;
-pcapService.getPackets(request).subscribe(r => packets = r)
-expect(packets).toBeTruthy()
-expect(packets.pdml).toBeTruthy()
-expect(packets.pdml.packet.length).toBe(1)
-expect(packets.pdml.packet[0].proto.length).toBeGreaterThan(3)
-
-console.log(packets)
- }))
-
-
-it('should ...', inject([PcapService], (service: PcapService) => {
- expect(service).toBeTruthy();
-}));
-
- })
-
-});
+mockBackend.connections.subscribe(connection => {
+ expect(connection.request.url).toMatch(
+/\/api\/v1\/pcap\/job_1234567890123_4567\/pdml\?page=1/
+ );
+ connection.mockRespond(
+new Response(new ResponseOptions({ body: pdmlJsonMock }))
+ );
+});
+pcapService.getPackets(jobId, 1).subscribe(r => (packets = r));
+expect(packets).toBeTruthy();
+expect(packets.pdml).toBeTruthy();
+expect(packets.pdml.packet.length).toBe(1);
+expect(packets.pdml.packet[0].protos.length).toBe(3);
+ }
+));
+ });
+ describe('pollStatus()', () => {
+it('should call getStatus() in intervals', fakeAsync(inject(
+ [PcapService, XHRBackend], (pcapService, mockBackend) => {
+ const responseMock: PcapStatusResponse = fakePcapStatusResponse;
+ const spy = spyOn(pcapService, 'getStatus').and.returnValue(
+Observable.of(responseMock)
+ );
+ let response;
+
+ pcapService.pollStatus(jobId).subscribe(r => (response = r));
+ tick(4000);
+ expect(spy.calls.count()).toBe(1);
+ tick(4000);
+ expect(spy.calls.count()).toBe(2);
+ discardPeriodicTasks();
+})
+ )
+);
+ });
+ describe('submitRequest()', () => {
+it('should return an Observable', inject(
+ [PcapService, XHRBackend],
+ (pcapService, mockBackend) => {
+const request: PcapRequest = fakePcapRequest;
+const responseMock: PcapStatusResponse = fakePcapStatusResponse;
+let response;
+
+mockBackend.connections.subscribe(connection => {
+
[44/51] [abbrv] metron git commit: METRON-1734 Src and Dst port filters are incorrect after changing to empty (merrimanr) closes apache/metron#1159
METRON-1734 Src and Dst port filters are incorrect after changing to empty
(merrimanr) closes apache/metron#1159
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ba0a6092
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ba0a6092
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ba0a6092
Branch: refs/heads/master
Commit: ba0a6092c97d23c118c2b5780c2559c2f849a220
Parents: 7a8c246
Author: merrimanr
Authored: Mon Aug 13 15:42:29 2018 -0500
Committer: rmerriman
Committed: Mon Aug 13 15:42:29 2018 -0500
--
.../pcap/pcap-filters/pcap-filters.component.spec.ts | 13 +
.../app/pcap/pcap-filters/pcap-filters.component.ts| 4
2 files changed, 17 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/ba0a6092/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
index c1d8664..3f1ab07 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
@@ -179,6 +179,19 @@ describe('PcapFiltersComponent', () => {
component.onSubmit();
});
+ it('Port fields should be removed from request when set to empty', () => {
+component.model.ipSrcPort = 44;
+component.model.ipDstPort = 44;
+component.ipSrcPort = '';
+component.ipDstPort = '';
+
+component.search.emit = (model: PcapRequest) => {
+ expect(model.ipSrcPort).toBeFalsy();
+ expect(model.ipDstPort).toBeFalsy();
+};
+component.onSubmit();
+ });
+
it('Filter should have an output called search', () => {
component.search.subscribe((filterModel) => {
expect(filterModel).toBeDefined();
http://git-wip-us.apache.org/repos/asf/metron/blob/ba0a6092/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
index 20a8c2b..df91ae3 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
@@ -63,9 +63,13 @@ export class PcapFiltersComponent implements OnInit,
OnChanges {
this.model.endTimeMs = new Date(this.endTimeStr).getTime();
if (this.ipSrcPort !== '') {
this.model.ipSrcPort = +this.ipSrcPort;
+} else {
+ delete this.model.ipSrcPort;
}
if (this.ipDstPort !== '') {
this.model.ipDstPort = +this.ipDstPort;
+} else {
+ delete this.model.ipDstPort;
}
this.search.emit(this.model);
[17/51] [abbrv] metron git commit: METRON-1691: REST should limit the number of Pcap jobs a user can submit (merrimanr via mmiklavc) closes apache/metron#1129
METRON-1691: REST should limit the number of Pcap jobs a user can submit
(merrimanr via mmiklavc) closes apache/metron#1129
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6c90724d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6c90724d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6c90724d
Branch: refs/heads/master
Commit: 6c90724d8d4f06c453128f860eff51037f1870d8
Parents: f1f5dda
Author: merrimanr
Authored: Wed Jul 25 10:58:05 2018 -0600
Committer: Michael Miklavcic
Committed: Wed Jul 25 10:58:05 2018 -0600
--
.../apache/metron/rest/model/pcap/Field.java| 36 ---
.../rest/model/pcap/FixedPcapRequest.java | 30 -
.../apache/metron/rest/model/pcap/Packet.java | 8 +--
.../metron/rest/model/pcap/PcapStatus.java | 24 +++
.../org/apache/metron/rest/model/pcap/Pdml.java | 21 +++---
.../apache/metron/rest/model/pcap/Proto.java| 24 +++
.../rest/model/pcap/QueryPcapRequest.java | 18 --
metron-interface/metron-rest/README.md | 27 ++--
.../apache/metron/rest/MetronRestConstants.java | 1 +
.../metron/rest/controller/PcapController.java | 13
.../apache/metron/rest/service/PcapService.java | 6 ++
.../rest/service/impl/PcapServiceImpl.java | 64 --
.../src/main/resources/application.yml | 1 +
.../PcapControllerIntegrationTest.java | 46 +
.../rest/service/impl/PcapServiceImplTest.java | 68
.../metron/job/manager/InMemoryJobManager.java | 4 ++
16 files changed, 261 insertions(+), 130 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
--
diff --git
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
index 9c2878b..4ed71c3 100644
---
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
+++
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
@@ -22,6 +22,7 @@ import
com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
public class Field {
@@ -132,33 +133,22 @@ public class Field {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
-
Field field = (Field) o;
-
-return (getName() != null ? getName().equals(field.getName()) :
field.getName() != null) &&
-(getPos() != null ? getPos().equals(field.getPos()) :
field.getPos() == null) &&
-(getShowname() != null ? getShowname().equals(field.getShowname())
: field.getShowname() == null) &&
-(getSize() != null ? getSize().equals(field.getSize()) :
field.getSize() == null) &&
-(getValue() != null ? getValue().equals(field.getValue()) :
field.getValue() == null) &&
-(getShow() != null ? getShow().equals(field.getShow()) :
field.getShow() == null) &&
-(getUnmaskedvalue() != null ?
getUnmaskedvalue().equals(field.getUnmaskedvalue()) : field.getUnmaskedvalue()
== null) &&
-(getHide() != null ? getHide().equals(field.getHide()) :
field.getHide() == null) &&
-(getFields() != null ? getFields().equals(field.getFields()) :
field.getFields() == null) &&
-(getProtos() != null ? getProtos().equals(field.getProtos()) :
field.getProtos() == null);
+return Objects.equals(name, field.name) &&
+Objects.equals(pos, field.pos) &&
+Objects.equals(showname, field.showname) &&
+Objects.equals(size, field.size) &&
+Objects.equals(value, field.value) &&
+Objects.equals(show, field.show) &&
+Objects.equals(unmaskedvalue, field.unmaskedvalue) &&
+Objects.equals(hide, field.hide) &&
+Objects.equals(fields, field.fields) &&
+Objects.equals(protos, field.protos);
}
@Override
public int hashCode() {
-int result = getName() != null ? getName().hashCode() : 0;
-result = 31 * result + (getPos() != null ? getPos().hashCode() : 0);
-result = 31 * result + (getShowname() != null ? getShowname().hashCode() :
0);
-result = 31 * result + (getSize() != null ? getSize().hashCode() : 0);
-result = 31 * result + (getValue() != null ? getValue().hashCode() : 0);
-result = 31 * result + (getShow() != null ? getShow().hashCode() : 0);
-result = 31 * result +
[10/51] [abbrv] metron git commit: METRON-1685 Retrieve Pcap results in raw binary format (merrimanr) closes apache/metron#1123
METRON-1685 Retrieve Pcap results in raw binary format (merrimanr) closes
apache/metron#1123
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a5a51399
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a5a51399
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a5a51399
Branch: refs/heads/master
Commit: a5a51399d2eafd2535d79bb13ee0d4d8eb2e2d23
Parents: 3e5ef41
Author: merrimanr
Authored: Fri Jul 20 09:37:34 2018 -0500
Committer: rmerriman
Committed: Fri Jul 20 09:37:34 2018 -0500
--
metron-interface/metron-rest/README.md | 10
.../metron/rest/controller/PcapController.java | 40 ++
.../apache/metron/rest/service/PcapService.java | 5 +-
.../rest/service/impl/PcapServiceImpl.java | 17 ++
.../PcapControllerIntegrationTest.java | 44
.../rest/service/impl/PcapServiceImplTest.java | 55
6 files changed, 170 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/README.md
--
diff --git a/metron-interface/metron-rest/README.md
b/metron-interface/metron-rest/README.md
index 7b3a263..4a7102f 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -256,6 +256,7 @@ Request and Response objects are JSON formatted. The JSON
schemas are available
| [ `GET /api/v1/pcap/fixed`](#get-apiv1pcapfixed)|
| [ `GET /api/v1/pcap/{jobId}`](#get-apiv1pcapjobid)|
| [ `GET /api/v1/pcap/{jobId}/pdml`](#get-apiv1pcapjobidpdml)|
+| [ `GET /api/v1/pcap/{jobId}/raw`](#get-apiv1pcapjobidraw)|
| [ `GET /api/v1/search/search`](#get-apiv1searchsearch)|
| [ `POST /api/v1/search/search`](#get-apiv1searchsearch)|
| [ `POST /api/v1/search/group`](#get-apiv1searchgroup)|
@@ -523,6 +524,15 @@ Request and Response objects are JSON formatted. The JSON
schemas are available
* Returns:
* 200 - Returns PDML in json format.
* 404 - Job or page is missing.
+
+### `POST /api/v1/pcap/{jobId}/raw`
+ * Description: Download Pcap Results for a page.
+ * Input:
+* jobId - Job ID of submitted job
+* page - Page number
+ * Returns:
+* 200 - Returns Pcap as a file download.
+* 404 - Job or page is missing.
### `POST /api/v1/search/search`
* Description: Searches the indexing store. GUIDs must be quoted to ensure
correct results.
http://git-wip-us.apache.org/repos/asf/metron/blob/a5a51399/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
index 47bc6a0..23bb0b9 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
@@ -21,6 +21,8 @@ import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.pcap.FixedPcapRequest;
import org.apache.metron.rest.model.pcap.PcapStatus;
@@ -37,10 +39,18 @@ import
org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
@RestController
@RequestMapping("/api/v1/pcap")
public class PcapController {
+ private static final String PCAP_FILENAME_FORMAT = "pcap_%s_%s.pcap";
+
@Autowired
private PcapService pcapQueryService;
@@ -99,4 +109,34 @@ public class PcapController {
}
}
+ @ApiOperation(value = "Download Pcap Results for a page.")
+ @ApiResponses(value = {
+ @ApiResponse(message = "Returns Pcap as a file download.", code =
200),
+ @ApiResponse(message = "Job or page is missing.", code = 404)
+ })
+ @RequestMapping(value = "/{jobId}/raw", method = RequestMethod.GET)
+ void raw(@ApiParam(name="jobId", value="Job ID of submitted job",
required=true)@PathVariable String jobId,
+ @ApiP
[12/51] [abbrv] metron git commit: METRON-1562 Enable Kerberos in REST for YARN and MR jobs (merrimanr) closes apache/metron#1094
METRON-1562 Enable Kerberos in REST for YARN and MR jobs (merrimanr) closes
apache/metron#1094
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8763fba2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8763fba2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8763fba2
Branch: refs/heads/master
Commit: 8763fba2ff579b37b6af82822fc4b2c4390665a8
Parents: b580a69
Author: merrimanr
Authored: Fri Jul 20 11:05:32 2018 -0500
Committer: rmerriman
Committed: Fri Jul 20 11:05:32 2018 -0500
--
.../ansible/roles/ambari_config/vars/single_node_vm.yml | 1 +
metron-interface/metron-rest/pom.xml | 10 ++
2 files changed, 11 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/8763fba2/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
--
diff --git
a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index 6b67719..a3c643b 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -69,6 +69,7 @@ configurations:
yarn_heapsize: 512
apptimelineserver_heapsize : 512
resourcemanager_heapsize: 1024
+ min_user_id: 500
- mapred-env:
jobhistory_heapsize: 256
- mapred-site:
http://git-wip-us.apache.org/repos/asf/metron/blob/8763fba2/metron-interface/metron-rest/pom.xml
--
diff --git a/metron-interface/metron-rest/pom.xml
b/metron-interface/metron-rest/pom.xml
index 1bf0fd6..e98c96b 100644
--- a/metron-interface/metron-rest/pom.xml
+++ b/metron-interface/metron-rest/pom.xml
@@ -191,6 +191,16 @@
+
+org.apache.hadoop
+hadoop-mapreduce-client-app
+${global_hadoop_version}
+
+
+org.apache.hadoop
+hadoop-mapreduce-client-core
+${global_hadoop_version}
+
org.apache.metron
metron-hbase-client
[13/51] [abbrv] metron git commit: METRON-1560 Update MPack to support Pcap panel (merrimanr) closes apache/metron#1124
METRON-1560 Update MPack to support Pcap panel (merrimanr) closes
apache/metron#1124
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e78bc530
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e78bc530
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e78bc530
Branch: refs/heads/master
Commit: e78bc530260aa622e8263cda2b94be99194dffb5
Parents: 8763fba
Author: merrimanr
Authored: Mon Jul 23 09:08:12 2018 -0500
Committer: rmerriman
Committed: Mon Jul 23 09:08:12 2018 -0500
--
.../CURRENT/configuration/metron-rest-env.xml | 25
.../package/scripts/params/params_linux.py | 5 +++-
.../CURRENT/package/scripts/rest_commands.py| 18 --
.../METRON/CURRENT/package/templates/metron.j2 | 4
.../METRON/CURRENT/themes/metron_theme.json | 10
.../src/main/config/rest_application.yml| 6 -
.../src/main/config/pcap.properties | 2 +-
7 files changed, 65 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index 066b527..20f9767 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -168,4 +168,29 @@
The field name where the threat triage score can be found
in the search indices. This setting primarily affects the Alerts
UI.
threat:triage:score
+
+pcap_base_path
+Pcap Base Path
+The HDFS path where pcap files are stored. These files
are the input to pcap queries.
+/apps/metron/pcap/input
+
+
+pcap_base_interim_result_path
+Pcap Base Interim Result Path
+The HDFS path where interim results of a pcap query are
stored, before they are separated into pages.
+/apps/metron/pcap/interim
+
+
+pcap_final_output_path
+Pcap Final Output Path
+The HDFS path where the final results of a pcap query are
stored.
+/apps/metron/pcap/output
+
+
+pcap_page_size
+Pcap Page Size
+The number of pcaps written to a page/file as a result of
a pcap query.
+10
+
+
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index b8d6123..73d3469 100755
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -386,7 +386,10 @@ bolt_hdfs_rotation_policy_units =
config['configurations']['metron-indexing-env'
bolt_hdfs_rotation_policy_count =
config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
# Pcap
-pcap_hdfs_dir = format("{metron_apps_hdfs_dir}/pcap")
+pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path']
+pcap_base_interim_result_path =
config['configurations']['metron-rest-env']['pcap_base_interim_result_path']
+pcap_final_output_path =
config['configurations']['metron-rest-env']['pcap_final_output_path']
+pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size']
pcap_configured_flag_file = status_params.pcap_configured_flag_file
# MapReduce
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resour
[02/51] [abbrv] metron git commit: METRON-1614: Create job status abstraction (mmiklavc via mmiklavc) closes apache/metron#1108
http://git-wip-us.apache.org/repos/asf/metron/blob/dbbf6243/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
index c7292ab..9ea7912 100644
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
+++
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
@@ -36,6 +36,7 @@ import java.util.Map;
import java.util.Properties;
import javax.annotation.Nullable;
import kafka.consumer.ConsumerIterator;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -45,6 +46,7 @@ import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.kafka.clients.producer.Producer;
import org.apache.metron.common.Constants;
+import org.apache.metron.common.utils.HDFSUtils;
import org.apache.metron.integration.BaseIntegrationTest;
import org.apache.metron.integration.ComponentRunner;
import org.apache.metron.integration.Processor;
@@ -55,12 +57,18 @@ import
org.apache.metron.integration.components.KafkaComponent;
import org.apache.metron.integration.components.MRComponent;
import org.apache.metron.integration.components.ZKServerComponent;
import org.apache.metron.integration.utils.KafkaUtil;
+import org.apache.metron.job.JobStatus;
+import org.apache.metron.job.Statusable;
import org.apache.metron.pcap.PacketInfo;
import org.apache.metron.pcap.PcapHelper;
import org.apache.metron.pcap.PcapMerger;
+import org.apache.metron.pcap.config.FixedPcapConfig;
+import org.apache.metron.pcap.config.PcapOptions;
import org.apache.metron.pcap.filter.fixed.FixedPcapFilter;
import org.apache.metron.pcap.filter.query.QueryPcapFilter;
+import org.apache.metron.pcap.finalizer.PcapFinalizerStrategies;
import org.apache.metron.pcap.mr.PcapJob;
+import org.apache.metron.pcap.query.PcapCli;
import org.apache.metron.spout.pcap.Endianness;
import org.apache.metron.spout.pcap.deserializer.Deserializers;
import org.apache.metron.test.utils.UnitTestHelper;
@@ -73,13 +81,22 @@ public class PcapTopologyIntegrationTest extends
BaseIntegrationTest {
final static String KAFKA_TOPIC = "pcap";
private static String BASE_DIR = "pcap";
private static String DATA_DIR = BASE_DIR + "/data_dir";
- private static String QUERY_DIR = BASE_DIR + "/query";
+ private static String INTERIM_RESULT = BASE_DIR + "/query";
+ private static String OUTPUT_DIR = BASE_DIR + "/output";
+ private static final int MAX_RETRIES = 30;
+ private static final int SLEEP_MS = 500;
private String topologiesDir = "src/main/flux";
private String targetDir = "target";
- private static void clearOutDir(File outDir) {
-for(File f : outDir.listFiles()) {
- f.delete();
+ private static void clearOutDirs(File... dirs) throws IOException {
+for(File dir: dirs) {
+ for(File f : dir.listFiles()) {
+if (f.isDirectory()) {
+ FileUtils.deleteDirectory(f);
+} else {
+ f.delete();
+}
+ }
}
}
private static int numFiles(File outDir, Configuration config) {
@@ -158,10 +175,10 @@ public class PcapTopologyIntegrationTest extends
BaseIntegrationTest {
topologiesDir = UnitTestHelper.findDir("topologies");
}
targetDir = UnitTestHelper.findDir("target");
-final File outDir = getOutDir(targetDir);
-final File queryDir = getQueryDir(targetDir);
-clearOutDir(outDir);
-clearOutDir(queryDir);
+final File inputDir = getDir(targetDir, DATA_DIR);
+final File interimResultDir = getDir(targetDir, INTERIM_RESULT);
+final File outputDir = getDir(targetDir, OUTPUT_DIR);
+clearOutDirs(inputDir, interimResultDir, outputDir);
File baseDir = new File(new File(targetDir), BASE_DIR);
//Assert.assertEquals(0, numFiles(outDir));
@@ -175,7 +192,7 @@ public class PcapTopologyIntegrationTest extends
BaseIntegrationTest {
setProperty("topology.worker.childopts", "");
setProperty("spout.kafka.topic.pcap", KAFKA_TOPIC);
setProperty("kafka.pcap.start", "EARLIEST");
- setProperty("kafka.pcap.out", outDir.getAbsolutePath());
+ setProperty("kafka.pcap.out", inputDir.getAbsolutePath());
setProperty("kafka.pcap.numPackets", "2");
setProperty("kafka.pcap.maxTimeMS", "2");
setProperty("kafka.pcap.ts_granularity", "NANOSECONDS");
@@ -219,7 +236,7 @@ public class PcapTopologyIntegrationTest extends
BaseIntegrationTest {
runner.process(new
[20/51] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
new file mode 100644
index 000..155e1f8
--- /dev/null
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
@@ -0,0 +1,1283 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { PcapPacketComponent } from './pcap-packet.component';
+import { PdmlPacket } from '../model/pdml';
+
+describe('PcapPacketComponent', () => {
+ let component: PcapPacketComponent;
+ let fixture: ComponentFixture;
+
+ beforeEach(async(() => {
+TestBed.configureTestingModule({
+ declarations: [ PcapPacketComponent ]
+})
+.compileComponents();
+ }));
+
+ beforeEach(() => {
+fixture = TestBed.createComponent(PcapPacketComponent);
+component = fixture.componentInstance;
+component.packet = fakePacket as PdmlPacket;
+fixture.detectChanges();
+ });
+
+ it('should create', () => {
+expect(component).toBeTruthy();
+ });
+});
+
+const fakePacket = {
+ name: "something",
+ expanded: false,
+ protos: [
+{
+ "name": "geninfo",
+ "pos": "0",
+ "showname": "General information",
+ "size": "722",
+ "hide": null,
+ "fields": [
+{
+ "name": "num",
+ "pos": "0",
+ "showname": "Number",
+ "size": "722",
+ "value": "1",
+ "show": "1",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "len",
+ "pos": "0",
+ "showname": "Frame Length",
+ "size": "722",
+ "value": "2d2",
+ "show": "722",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "caplen",
+ "pos": "0",
+ "showname": "Captured Length",
+ "size": "722",
+ "value": "2d2",
+ "show": "722",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "timestamp",
+ "pos": "0",
+ "showname": "Captured Time",
+ "size": "722",
+ "value": "1458240269.373968000",
+ "show": "Mar 17, 2016 18:44:29.373968000 UTC",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+}
+ ]
+},
+{
+ "name": "frame",
+ "pos": "0",
+ "showname": "Frame 1: 722 bytes on wire (5776 bits), 722 bytes captured
(5776 bits)",
+ "size": "722",
+ "hide": null,
+ "fields": [
+{
+ "name": "frame.dlt",
+ "pos": "0",
+ "showname": "WTAP_ENCAP: 1",
+ "size": "0",
+ "value": null,
+ "show": "1",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "frame.time",
+ "pos": "0",
+ "showname": "Arrival Time: Mar 17, 2016 18:44:29.373968000 UTC",
+ "size": "0",
+ "value": null,
+ "show": "Mar 17, 2016 18:44:29.373968000",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "frame.offset_shift",
+ "pos": "0",
+ "showname": "Time shift for this packet: 0.0 seconds",
+ "size": "0",
+ "value": null,
+ "show": "0.0",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "frame.time_epoch",
+ "pos": "0",
+
[25/51] [abbrv] metron git commit: METRON-1662 PCAP UI - Downloading PCAP page files (tiborm via merrimanr) closes apache/metron#1118
METRON-1662 PCAP UI - Downloading PCAP page files (tiborm via merrimanr) closes
apache/metron#1118
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b445bfe2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b445bfe2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b445bfe2
Branch: refs/heads/master
Commit: b445bfe242123e544c1fef41e2ab5bdc07dbc56e
Parents: 2b6959b
Author: tiborm
Authored: Fri Aug 3 08:13:11 2018 -0500
Committer: rmerriman
Committed: Fri Aug 3 08:13:11 2018 -0500
--
.../metron-alerts/src/app/pcap/model/pdml.ts| 12 +++-
.../pcap/pcap-packet/pcap-packet.component.ts | 9 ++
.../pcap/pcap-panel/pcap-panel.component.html | 2 +-
.../app/pcap/pcap-panel/pcap-panel.component.ts | 30 +++-
.../src/app/pcap/service/pcap.service.ts| 15 ++
5 files changed, 33 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/b445bfe2/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
index 97fc347..34245c6 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
@@ -17,8 +17,7 @@
*/
export class PdmlField {
- //public $: any[]
-public name: string;
+ public name: string;
public fields?: PdmlField[]
public show: string;
public value: string;
@@ -26,9 +25,8 @@
}
export class PdmlProto {
- //public $: any
-public name: string;
-public showname: string;
+ public name: string;
+ public showname: string;
public fields: PdmlField[]
public static findField(p: PdmlProto, name: string): PdmlField {
@@ -36,10 +34,8 @@ export class PdmlProto {
}
}
-
export class PdmlPacket {
- //public $: any
-public name: string;
+ public name: string;
public protos: PdmlProto[]
public expanded: boolean = false
}
http://git-wip-us.apache.org/repos/asf/metron/blob/b445bfe2/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
index f1bbe2d..ca28a39 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
@@ -24,16 +24,11 @@ import { PdmlPacket } from '../model/pdml'
templateUrl: './pcap-packet.component.html',
styleUrls: ['./pcap-packet.component.scss']
})
-export class PcapPacketComponent implements OnInit {
+export class PcapPacketComponent {
@Input() packet: PdmlPacket
- constructor() { }
-
- ngOnInit() {
- }
-
toggle() {
-this.packet.expanded = !this.packet.expanded
+this.packet.expanded = !this.packet.expanded;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/b445bfe2/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
index b373914..1fd250f 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
@@ -27,5 +27,5 @@
-
+ Download PCAP
http://git-wip-us.apache.org/repos/asf/metron/blob/b445bfe2/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
index fd49ec7..4114e24 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
@@ -15,47 +15,45 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { Component, OnInit, Input } from '@angular/core';
+import { Component, Input } from '@angular/core';
-import { PcapService, PcapStatusResponse } from '../service/pcap.service'
-import { PcapRequest } from '../model/pcap.request'
-import { Pdml } from '../model/pdml'
-import
[05/51] [abbrv] metron git commit: METRON-1614: Create job status abstraction (mmiklavc via mmiklavc) closes apache/metron#1108
METRON-1614: Create job status abstraction (mmiklavc via mmiklavc) closes apache/metron#1108 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/dbbf6243 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/dbbf6243 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/dbbf6243 Branch: refs/heads/master Commit: dbbf624367f8a3fb5d78352be0d9a2dec0533923 Parents: f30e9cc Author: mmiklavc Authored: Wed Jul 18 15:48:08 2018 -0600 Committer: Michael Miklavcic Committed: Wed Jul 18 15:48:08 2018 -0600 -- metron-deployment/amazon-ec2/conf/defaults.yml | 1 - metron-deployment/amazon-ec2/playbook.yml | 2 +- .../ansible/playbooks/metron_install.yml| 7 - .../roles/metron_pcapservice/defaults/main.yml | 28 - .../roles/metron_pcapservice/meta/main.yml | 19 - .../metron_pcapservice/tasks/config-hbase.yml | 26 - .../roles/metron_pcapservice/tasks/main.yml | 25 - .../metron_pcapservice/tasks/pcapservice.yml| 30 - .../templates/metron-pcapservice-logrotate.yml | 27 - .../metron_pcapservice/templates/pcapservice| 84 --- .../ubuntu14/ansible/inventory/hosts| 4 - metron-interface/metron-rest-client/pom.xml | 5 + .../metron/rest/model/pcap/PcapRequest.java | 37 +- .../metron/rest/controller/PcapController.java | 32 +- .../rest/service/impl/PcapServiceImpl.java | 11 +- .../rest/service/impl/PcapServiceImplTest.java | 32 +- metron-platform/metron-api/README.md| 65 -- metron-platform/metron-api/pom.xml | 269 - .../apache/metron/api/ConfigurationManager.java | 135 - .../api/helper/service/PcapServiceCli.java | 169 -- .../metron/pcapservice/ConfigurationUtil.java | 64 -- .../pcapservice/PcapReceiverImplRestEasy.java | 299 - .../metron/pcapservice/PcapsResponse.java | 118 .../metron/pcapservice/RestTestingUtil.java | 329 -- .../pcapservice/rest/JettyServiceRunner.java| 44 -- .../metron/pcapservice/rest/PcapService.java| 56 -- .../src/main/resources/META-INF/LICENSE | 605 --- .../src/main/resources/META-INF/NOTICE | 39 -- .../pcapservice/ConfigurationUtilTest.java | 43 -- .../PcapReceiverImplRestEasyTest.java | 255 .../src/test/resources/log4j.properties | 24 - .../src/test/resources/test-tcp-packet.pcap | Bin 144 -> 0 bytes .../common/configuration/ConfigOption.java | 45 ++ .../apache/metron/common/utils/HDFSUtils.java | 20 +- metron-platform/metron-job/README.md| 26 + .../metron-job_state_statechart_diagram.svg | 14 + .../metron-job_state_statechart_diagram.xml | 14 + .../java/org/apache/metron/job/Finalizer.java | 38 ++ .../org/apache/metron/job/JobException.java | 31 + .../java/org/apache/metron/job/JobStatus.java | 17 +- .../java/org/apache/metron/job/Pageable.java| 16 +- .../java/org/apache/metron/job/Statusable.java | 32 +- .../metron/job/manager/InMemoryJobManager.java | 82 +++ .../apache/metron/job/manager/JobManager.java | 41 ++ .../job/manager/InMemoryJobManagerTest.java | 192 ++ metron-platform/metron-pcap-backend/pom.xml | 6 + .../org/apache/metron/pcap/query/CliConfig.java | 128 .../org/apache/metron/pcap/query/CliParser.java | 24 +- .../metron/pcap/query/FixedCliConfig.java | 50 -- .../metron/pcap/query/FixedCliParser.java | 10 +- .../org/apache/metron/pcap/query/PcapCli.java | 89 ++- .../metron/pcap/query/QueryCliConfig.java | 34 -- .../metron/pcap/query/QueryCliParser.java | 10 +- .../org/apache/metron/pcap/PcapJobTest.java | 189 +++--- .../PcapTopologyIntegrationTest.java| 545 ++--- .../apache/metron/pcap/query/PcapCliTest.java | 170 +++--- .../java/org/apache/metron/pcap/PcapFiles.java | 42 -- .../java/org/apache/metron/pcap/PcapPages.java | 86 +++ .../metron/pcap/config/FixedPcapConfig.java | 46 ++ .../apache/metron/pcap/config/PcapConfig.java | 131 .../apache/metron/pcap/config/PcapOptions.java | 66 ++ .../metron/pcap/config/QueryPcapConfig.java | 33 + .../metron/pcap/finalizer/PcapCliFinalizer.java | 45 ++ .../metron/pcap/finalizer/PcapFinalizer.java| 128 .../pcap/finalizer/PcapFinalizerStrategies.java | 47 ++ .../pcap/finalizer/PcapRestFinalizer.java | 44 ++ .../java/org/apache/metron/pcap/mr/PcapJob.java | 337 +++ .../metron/pcap/writer/PcapResultsWriter.java | 59 ++ .../metron/pcap/writer/ResultsWriter.java | 59 -- .../org/apache/metron/pcap/PcapPagesTest.java | 76 +++ metron-platform/pom.xml | 1 - 71 files changed, 2159 insertions(+), 3748 deletions(-) --
[04/51] [abbrv] metron git commit: METRON-1614: Create job status abstraction (mmiklavc via mmiklavc) closes apache/metron#1108
http://git-wip-us.apache.org/repos/asf/metron/blob/dbbf6243/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/RestTestingUtil.java
--
diff --git
a/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/RestTestingUtil.java
b/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/RestTestingUtil.java
deleted file mode 100644
index ef31a2a..000
---
a/metron-platform/metron-api/src/main/java/org/apache/metron/pcapservice/RestTestingUtil.java
+++ /dev/null
@@ -1,329 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.pcapservice;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.springframework.http.HttpEntity;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpMethod;
-import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.client.RestTemplate;
-
-/**
- * The Class RestTestingUtil.
- */
-public class RestTestingUtil {
-
- /** The host name. */
- public static String hostName = null;
-
- /**
- * Gets the pcaps by keys.
- *
- * @param keys
- * the keys
- * @return the pcaps by keys
- */
- @SuppressWarnings("unchecked")
- private static void getPcapsByKeys(String keys) {
-System.out
-.println("**getPcapsByKeys
**");
-// 1.
-String url = "http://; + hostName
-+ "/cisco-rest/pcapGetter/getPcapsByKeys?keys={keys}"
-+ "={includeReverseTraffic}"
-+ "={startTime}" + "={endTime}"
-+ "={maxResponseSize}";
-// default values
-String startTime = "-1";
-String endTime = "-1";
-String maxResponseSize = "6";
-String includeReverseTraffic = "false";
-
-@SuppressWarnings("rawtypes")
-Map map = new HashMap();
-map.put("keys", keys);
-map.put("includeReverseTraffic", includeReverseTraffic);
-map.put("startTime", startTime);
-map.put("endTime", endTime);
-map.put("maxResponseSize", maxResponseSize);
-
-RestTemplate template = new RestTemplate();
-
-// set headers and entity to send
-HttpHeaders headers = new HttpHeaders();
-headers.set("Accept", MediaType.APPLICATION_OCTET_STREAM_VALUE);
-HttpEntity requestEntity = new HttpEntity(headers);
-
-// 1.
-ResponseEntity response1 = template.exchange(url, HttpMethod.GET,
-requestEntity, byte[].class, map);
-System.out
-
.println("");
-System.out
-.format(
-"getPcapsByKeys : request= \n response= %s \n",
-keys, includeReverseTraffic, startTime, endTime, maxResponseSize,
-response1);
-System.out
-
.println("");
-System.out.println();
-
-// 2. with reverse traffic
-includeReverseTraffic = "true";
-map.put("includeReverseTraffic", includeReverseTraffic);
-ResponseEntity response2 = template.exchange(url, HttpMethod.GET,
-requestEntity, byte[].class, map);
-System.out
-
.println("");
-System.out
-.format(
-"getPcapsByKeys : request= \n response= %s \n",
-keys, includeReverseTraffic, startTime, endTime, maxResponseSize,
-response2);
-System.out
-
.println("");
-System.out.println();
-
-// 3.with time range
-startTime = System.getProperty("startTime", "-1");
-endTime = System.getProperty("endTime", "-1");
-map.put("startTime", startTime);
-map.put("endTime", endTime);
-ResponseEntity response3 = template.exchange(url, HttpMethod.GET,
-requestEntity, byte[].class, map);
-System.out
-
[01/51] [abbrv] metron git commit: METRON-1614: Create job status abstraction (mmiklavc via mmiklavc) closes apache/metron#1108
Repository: metron
Updated Branches:
refs/heads/master 308c2b27b -> 9fdccba37
http://git-wip-us.apache.org/repos/asf/metron/blob/dbbf6243/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
index 269f69b..05c494b 100644
---
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
+++
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
@@ -22,17 +22,17 @@ import static
org.apache.metron.pcap.PcapHelper.greaterThanOrEqualTo;
import static org.apache.metron.pcap.PcapHelper.lessThanOrEqualTo;
import com.google.common.base.Joiner;
-import com.google.common.collect.Iterables;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Timer;
+import java.util.TimerTask;
import java.util.UUID;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
@@ -50,30 +50,43 @@ import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
-import org.apache.metron.common.hadoop.SequenceFileIterable;
+import org.apache.metron.job.Finalizer;
+import org.apache.metron.job.JobException;
import org.apache.metron.job.JobStatus;
import org.apache.metron.job.JobStatus.State;
import org.apache.metron.job.Pageable;
import org.apache.metron.job.Statusable;
import org.apache.metron.pcap.PacketInfo;
-import org.apache.metron.pcap.PcapFiles;
import org.apache.metron.pcap.PcapHelper;
+import org.apache.metron.pcap.PcapPages;
+import org.apache.metron.pcap.config.PcapOptions;
import org.apache.metron.pcap.filter.PcapFilter;
import org.apache.metron.pcap.filter.PcapFilterConfigurator;
import org.apache.metron.pcap.filter.PcapFilters;
import org.apache.metron.pcap.utils.FileFilterUtil;
-import org.apache.metron.pcap.writer.ResultsWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class PcapJob implements Statusable {
+/**
+ * Encompasses MapReduce job and final writing of Pageable results to
specified location.
+ * Cleans up MapReduce results from HDFS on completion.
+ */
+public class PcapJob implements Statusable {
private static final Logger LOG =
LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String START_TS_CONF = "start_ts";
public static final String END_TS_CONF = "end_ts";
public static final String WIDTH_CONF = "width";
- private Job job; // store a running MR job reference for async status check
- private Path outputPath;
+ private static final long THREE_SECONDS = 3000;
+ private static final long ONE_SECOND = 1000;
+ private Job mrJob; // store a running MR job reference for async status check
+ private State jobState; // overall job state, including finalization step
+ private Finalizer finalizer;
+ private Map configuration;
+ private Pageable finalResults;
+ private Timer timer;
+ private long statusInterval; // how often timer thread checks job status.
+ private long completeCheckInterval; // how long we sleep between isDone
checks in get()
public static enum PCAP_COUNTER {
MALFORMED_PACKET_COUNT
@@ -167,61 +180,92 @@ public class PcapJob implements Statusable {
}
}
+ public PcapJob() {
+jobState = State.NOT_RUNNING;
+finalResults = new PcapPages();
+statusInterval = THREE_SECONDS;
+completeCheckInterval = ONE_SECOND;
+ }
+
/**
- * Run query synchronously.
+ * Primarily for testing.
+ *
+ * @param interval time in millis
*/
- public SequenceFileIterable query(Path basePath
-, Path baseOutputPath
-, long beginNS
-, long endNS
-, int numReducers
-, T fields
-, Configuration conf
-, FileSystem fs
-, PcapFilterConfigurator filterImpl
-) throws IOException, ClassNotFoundException,
InterruptedException {
-Statusable statusable = query(Optional.empty(), basePath, baseOutputPath,
beginNS, endNS, numReducers, fields,
-conf,
-fs, filterImpl, true);
-JobStatus jobStatus = statusable.getStatus();
-if (jobStatus.getState() == State.SUCCEEDED) {
- Path resultPath = jobStatus.getResultPath();
- return
[14/51] [abbrv] metron git commit: METRON-1661 Create Pcap Query Filter endpoint (merrimanr) closes apache/metron#1125
METRON-1661 Create Pcap Query Filter endpoint (merrimanr) closes
apache/metron#1125
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d0e87576
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d0e87576
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d0e87576
Branch: refs/heads/master
Commit: d0e87576944b6370eb1d9a0c98d0450f2c51f052
Parents: e78bc53
Author: merrimanr
Authored: Mon Jul 23 09:09:37 2018 -0500
Committer: rmerriman
Committed: Mon Jul 23 09:09:37 2018 -0500
--
.../rest/model/pcap/FixedPcapRequest.java | 1 +
.../metron/rest/model/pcap/PcapRequest.java | 4 +-
.../rest/model/pcap/QueryPcapOptions.java | 35 +
.../rest/model/pcap/QueryPcapRequest.java | 59 ++
metron-interface/metron-rest/README.md | 10 ++-
.../metron/rest/controller/PcapController.java | 15 +++-
.../apache/metron/rest/service/PcapService.java | 8 +-
.../rest/service/impl/PcapServiceImpl.java | 13 ++--
.../PcapControllerIntegrationTest.java | 81 +---
.../apache/metron/rest/mock/MockPcapJob.java| 13 ++--
.../rest/service/impl/PcapServiceImplTest.java | 55 +++--
11 files changed, 240 insertions(+), 54 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
--
diff --git
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
index a2d345b..d91aac7 100644
---
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
+++
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
@@ -87,6 +87,7 @@ public class FixedPcapRequest extends PcapRequest {
FixedPcapOptions.INCLUDE_REVERSE.put(this, includeReverse);
}
+ @Override
public void setFields() {
Map fields = new HashMap<>();
if (getIpSrcAddr() != null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
--
diff --git
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
index 64ed932..44721fa 100644
---
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
+++
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
@@ -22,7 +22,7 @@ import org.apache.metron.pcap.config.PcapOptions;
import java.util.HashMap;
-public class PcapRequest extends AbstractMapDecorator {
+public abstract class PcapRequest extends AbstractMapDecorator
{
public PcapRequest() {
super(new HashMap<>());
@@ -78,4 +78,6 @@ public class PcapRequest extends AbstractMapDecorator {
public void setNumReducers(Integer numReducers) {
PcapOptions.NUM_REDUCERS.put(this, numReducers);
}
+
+ public abstract void setFields();
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
--
diff --git
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
new file mode 100644
index 000..1ae3da6
--- /dev/null
+++
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY
[06/51] [abbrv] metron git commit: METRON-1674 Create REST endpoint for job status abstraction (merrimanr) closes apache/metron#1109
http://git-wip-us.apache.org/repos/asf/metron/blob/39ae9f46/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java
index 059bba2..95907df 100644
---
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java
+++
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapRestFinalizer.java
@@ -18,27 +18,27 @@
package org.apache.metron.pcap.finalizer;
-import java.util.Map;
import org.apache.hadoop.fs.Path;
+import org.apache.metron.job.Statusable;
import org.apache.metron.pcap.config.PcapOptions;
+import java.util.Map;
+
/**
* Write to HDFS.
*/
public class PcapRestFinalizer extends PcapFinalizer {
- /**
- * Format will have the format output-path/page-page-num.pcap
- * The filename prefix is pluggable, but in most cases it will be provided
via the PcapConfig
- * as a formatted timestamp + uuid. A final sample format will look as
follows:
- *
/base/output/path/pcap-data-201807181911-09855b4ae3204dee8b63760d65198da3+0001.pcap
- */
- private static final String PCAP_CLI_FILENAME_FORMAT = "%s/page-%s.pcap";
+ private static final String PCAP_REST_FILEPATH_FORMAT =
"%s/%s/%s/%s/page-%s.pcap";
+
+ private String jobType = Statusable.JobType.MAP_REDUCE.name();
@Override
- protected String getOutputFileName(Map config, int
partition) {
-Path finalOutputPath =
PcapOptions.FINAL_OUTPUT_PATH.getTransformed(config, Path.class);
-return String.format(PCAP_CLI_FILENAME_FORMAT, finalOutputPath, partition);
+ protected Path getOutputPath(Map config, int partition) {
+String finalOutputPath = PcapOptions.FINAL_OUTPUT_PATH.get(config,
String.class);
+String user = PcapOptions.USERNAME.get(config, String.class);
+String jobId = PcapOptions.JOB_ID.get(config, String.class);
+return new Path(String.format(PCAP_REST_FILEPATH_FORMAT, finalOutputPath,
user, jobType, jobId, partition));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/39ae9f46/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
index 05c494b..1dd670d 100644
---
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
+++
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/mr/PcapJob.java
@@ -215,14 +215,24 @@ public class PcapJob implements Statusable {
FileSystem fileSystem = PcapOptions.FILESYSTEM.get(configuration,
FileSystem.class);
Path basePath = PcapOptions.BASE_PATH.getTransformed(configuration,
Path.class);
Path baseInterimResultPath =
PcapOptions.BASE_INTERIM_RESULT_PATH.getTransformed(configuration, Path.class);
-long startTime = PcapOptions.START_TIME_NS.get(configuration, Long.class);
-long endTime = PcapOptions.END_TIME_NS.get(configuration, Long.class);
+long startTime;
+if (configuration.containsKey(PcapOptions.START_TIME_NS.getKey())) {
+ startTime = PcapOptions.START_TIME_NS.get(configuration, Long.class);
+} else {
+ startTime = PcapOptions.START_TIME_MS.get(configuration, Long.class) *
100;
+}
+long endTime;
+if (configuration.containsKey(PcapOptions.END_TIME_NS.getKey())) {
+ endTime = PcapOptions.END_TIME_NS.get(configuration, Long.class);
+} else {
+ endTime = PcapOptions.END_TIME_MS.get(configuration, Long.class) *
100;
+}
int numReducers = PcapOptions.NUM_REDUCERS.get(configuration,
Integer.class);
T fields = (T) PcapOptions.FIELDS.get(configuration, Object.class);
PcapFilterConfigurator filterImpl =
PcapOptions.FILTER_IMPL.get(configuration, PcapFilterConfigurator.class);
try {
- return query(jobName,
+ Statusable statusable = query(jobName,
basePath,
baseInterimResultPath,
startTime,
@@ -233,6 +243,8 @@ public class PcapJob implements Statusable {
new Configuration(hadoopConf),
fileSystem,
filterImpl);
+ PcapOptions.JOB_ID.put(configuration, statusable.getStatus().getJobId());
+ return statusable;
} catch (IOException | InterruptedException | ClassNotFoundException e) {
throw new JobException("Failed to run pcap query.", e);
}
[19/51] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
new file mode 100644
index 000..244a3ea
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
@@ -0,0 +1,1752 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { TestBed, async, inject } from '@angular/core/testing';
+import {
+ BaseRequestOptions,
+ HttpModule,
+ Http,
+ Response,
+ ResponseOptions
+} from '@angular/http';
+import { MockBackend } from '@angular/http/testing';
+
+import { PcapService } from './pcap.service';
+import { PcapRequest } from '../model/pcap.request';
+
+describe('PcapService', () => {
+ beforeEach(() => {
+
+TestBed.configureTestingModule({
+ imports: [HttpModule],
+ providers: [
+PcapService,
+{
+ provide: Http,
+ useFactory: (mockBackend, options) => {
+return new Http(mockBackend, options);
+ },
+ deps: [MockBackend, BaseRequestOptions]
+},
+MockBackend,
+BaseRequestOptions
+ ]
+});
+ });
+ describe('getPackets()', () => {
+it('should return an Observable',
+ inject([PcapService, MockBackend], (pcapService, mockBackend) => {
+
+let request: PcapRequest = {
+ startTimeMs: 0,
+ endTimeMs: 0,
+ ipSrcAddr: '0.0.0.0',
+ ipSrcPort: '80',
+ ipDstAddr: '0.0.0.0',
+ ipDstPort: '80',
+ protocol: '*',
+ packetFilter: '*',
+ includeReverse: false,
+};
+
+mockBackend.connections.subscribe((connection) => {
+ connection.mockRespond(new Response(new ResponseOptions({body:
pdml_json()})));
+});
+let packets;
+pcapService.getPackets(request).subscribe(r => packets = r)
+expect(packets).toBeTruthy()
+expect(packets.pdml).toBeTruthy()
+expect(packets.pdml.packet.length).toBe(1)
+expect(packets.pdml.packet[0].proto.length).toBeGreaterThan(3)
+
+console.log(packets)
+ }))
+
+
+it('should ...', inject([PcapService], (service: PcapService) => {
+ expect(service).toBeTruthy();
+}));
+
+ })
+
+});
+
+
+
+
+function pdml_json() {
+ return `{
+ "pdml": {
+"$": {
+ "version": "0",
+ "creator": "wireshark/2.4.2",
+ "time": "Tue Mar 27 21:55:25 2018",
+ "capture_file":
"./metron-platform/metron-api/src/test/resources/test-tcp-packet.pcap"
+},
+"packet": [
+ {
+"proto": [
+ {
+"$": {
+ "name": "geninfo",
+ "pos": "0",
+ "showname": "General information",
+ "size": "104"
+},
+"field": [
+ {
+"$": {
+ "name": "num",
+ "pos": "0",
+ "show": "1",
+ "showname": "Number",
+ "value": "1",
+ "size": "104"
+}
+ },
+ {
+"$": {
+ "name": "len",
+ "pos": "0",
+ "show": "104",
+ "showname": "Frame Length",
+ "value": "68",
+ "size": "104"
+}
+ },
+ {
+"$": {
+ "name": "caplen",
+ "pos": "0",
+ "show": "104",
+ "showname": "Captured Length",
+ "value": "68",
+ "size": "104"
+}
+ },
+ {
+"$": {
+ "name": "timestamp",
+ "pos": "0",
+ "show": "Mar 26, 2014 19:59:40.024362000 GMT",
+ "showname": "Captured Time",
+ "value": "1395863980.024362000",
+ "size": "104"
+}
+
metron git commit: METRON-1739 UDP packets are not handled (merrimanr) closes apache/metron#1168
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1554-pcap-query-panel 076a6a197 -> 14e80b3c2
METRON-1739 UDP packets are not handled (merrimanr) closes apache/metron#1168
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/14e80b3c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/14e80b3c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/14e80b3c
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 14e80b3c229ea2e5a8d615c87d822d4b44d8ec98
Parents: 076a6a1
Author: merrimanr
Authored: Fri Aug 17 08:47:32 2018 -0500
Committer: rmerriman
Committed: Fri Aug 17 08:47:32 2018 -0500
--
.../src/app/pcap/model/pdml.mock.ts | 31
.../pcap-packet-line.component.spec.ts | 53 --
.../pcap-packet-line.component.ts | 15 ++--
.../java/org/apache/metron/pcap/PcapHelper.java | 18 -
.../org/apache/metron/pcap/PcapHelperTest.java | 76
5 files changed, 180 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/14e80b3c/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
index 6316b92..2520da1 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
@@ -60,3 +60,34 @@ export const fakePacket = {
} as PdmlProto
]
} as PdmlPacket;
+
+export const fakeUdpPacket = {
+ "name": '',
+ "expanded": false,
+ "protos": [
+{
+ "name": "geninfo",
+ "showname": "",
+ "fields": [
+{ "name": "timestamp", "pos": "0", "showname": "Captured Time",
"size": "342", "value": "1534414874.932794000", "show": "Aug 16, 2018
10:21:14.932794000 UTC", "unmaskedvalue": null, "hide": null, "fields": null,
"protos": null } as PdmlField
+ ]
+} as PdmlProto,
+{
+ "name": "ip",
+ "showname": "",
+ "fields": [
+{ "name": "ip.proto", "pos": "23", "showname": "Protocol: UDP (17)",
"size": "1", "value": "11", "show": "17", "unmaskedvalue": null, "hide": null,
"fields": null, "protos": null } as PdmlField,
+{ "name": "ip.src", "pos": "26", "showname": "Source: 0.0.0.0
(0.0.0.0)", "size": "4", "value": "", "show": "0.0.0.0",
"unmaskedvalue": null, "hide": null, "fields": null, "protos": null } as
PdmlField,
+{ "name": "ip.dst", "pos": "30", "showname": "Destination:
255.255.255.255 (255.255.255.255)", "size": "4", "value": "", "show":
"255.255.255.255", "unmaskedvalue": null, "hide": null, "fields": null,
"protos": null } as PdmlField
+ ]
+} as PdmlProto,
+{
+ "name": "udp",
+ "showname": "User Datagram Protocol, Src Port: bootpc (68), Dst Port:
bootps (67)",
+ "fields": [
+{ "name": "udp.srcport", "pos": "34", "showname": "Source port: bootpc
(68)", "size": "2", "value": "0044", "show": "68", "unmaskedvalue": null,
"hide": null, "fields": null, "protos": null } as PdmlField,
+{ "name": "udp.dstport", "pos": "36", "showname": "Destination port:
bootps (67)", "size": "2", "value": "0043", "show": "67", "unmaskedvalue":
null, "hide": null, "fields": null, "protos": null } as PdmlField
+ ],
+} as PdmlProto
+ ]
+} as PdmlPacket;
http://git-wip-us.apache.org/repos/asf/metron/blob/14e80b3c/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
--
metron git commit: METRON-1735 Empty print status option causes NPE (merrimanr) closes apache/metron#1160
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1554-pcap-query-panel bce9b903f -> 8a926dd5a
METRON-1735 Empty print status option causes NPE (merrimanr) closes
apache/metron#1160
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8a926dd5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8a926dd5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8a926dd5
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 8a926dd5a4501df8c7ef51b526ad5e4d752a5e19
Parents: bce9b90
Author: merrimanr
Authored: Tue Aug 14 12:59:38 2018 -0500
Committer: rmerriman
Committed: Tue Aug 14 12:59:38 2018 -0500
--
.../org/apache/metron/pcap/query/CliParser.java | 4
.../apache/metron/pcap/query/PcapCliTest.java| 7 ++-
.../apache/metron/pcap/config/PcapConfig.java| 2 +-
.../java/org/apache/metron/pcap/mr/PcapJob.java | 18 +-
.../org/apache/metron/pcap/mr/PcapJobTest.java | 19 +++
5 files changed, 35 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index 5040f90..b9a2a50 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -55,7 +55,6 @@ public class CliParser {
options.addOption(newOption("rpf", "records_per_file", true,
String.format("Number of records to include in each output pcap file (defaults
to %s)", NUM_RECORDS_PER_FILE_DEFAULT)));
options.addOption(newOption("et", "end_time", true, "Packet end time
range. Default is current system time."));
options.addOption(newOption("df", "date_format", true, "Date format to use
for parsing start_time and end_time. Default is to use time in millis since the
epoch."));
-options.addOption(newOption("ps", "print_status", false, "Print the status
of the job as it runs"));
options.addOption(newOption("yq", "yarn_queue", true, "Yarn queue this job
will be submitted to"));
return options;
}
@@ -127,9 +126,6 @@ public class CliParser {
//no-op
}
}
-if (commandLine.hasOption("print_status")) {
- config.setPrintJobStatus(true);
-}
if (commandLine.hasOption("yarn_queue")) {
config.setYarnQueue(commandLine.getOptionValue("yarn_queue"));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index a71e997..3e7aad2 100644
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -172,8 +172,7 @@ public class PcapCliTest {
"-protocol", "6",
"-include_reverse",
"-num_reducers", "10",
-"-records_per_file", "1000",
-"-ps"
+"-records_per_file", "1000"
};
Map query = new HashMap() {{
put(Constants.Fields.SRC_ADDR.getName(), "192.168.1.1");
@@ -217,7 +216,6 @@ public class PcapCliTest {
"-include_reverse",
"-num_reducers", "10",
"-records_per_file", "1000",
-"-ps",
"-yq", "pcap"
};
Map query = new HashMap() {{
@@ -295,8 +293,7 @@ public class PcapCliTest {
"-base_path", "/base/path",
"-base_output_path", "/base/output/path",
"-query", "some query string",
-"-records_per_file", "1000",
-"-ps"
+"-records_per_file", &quo
metron git commit: METRON-1733 PCAP UI - PCAP queries don t work on Safari (sardell via merrimanr) closes apache/metron#1158
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1554-pcap-query-panel ba0a6092c -> bce9b903f
METRON-1733 PCAP UI - PCAP queries dont work on Safari (sardell via
merrimanr) closes apache/metron#1158
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bce9b903
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bce9b903
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bce9b903
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: bce9b903fbd3436ea0c41eb8d75da279e16ce922
Parents: ba0a609
Author: sardell
Authored: Mon Aug 13 16:04:45 2018 -0500
Committer: rmerriman
Committed: Mon Aug 13 16:04:45 2018 -0500
--
.../src/app/pcap/pcap-filters/pcap-filters.component.ts | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/bce9b903/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
index df91ae3..b23a2e2 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
@@ -59,8 +59,8 @@ export class PcapFiltersComponent implements OnInit,
OnChanges {
}
onSubmit() {
-this.model.startTimeMs = new Date(this.startTimeStr).getTime();
-this.model.endTimeMs = new Date(this.endTimeStr).getTime();
+this.model.startTimeMs = moment(this.startTimeStr,
DEFAULT_TIMESTAMP_FORMAT).valueOf();
+this.model.endTimeMs = moment(this.endTimeStr,
DEFAULT_TIMESTAMP_FORMAT).valueOf();
if (this.ipSrcPort !== '') {
this.model.ipSrcPort = +this.ipSrcPort;
} else {
metron git commit: METRON-1725 Add ability to specify YARN queue for pcap jobs (merrimanr) closes apache/metron#1153
Repository: metron Updated Branches: refs/heads/feature/METRON-1554-pcap-query-panel 73dc63e67 -> 7a8c24674 METRON-1725 Add ability to specify YARN queue for pcap jobs (merrimanr) closes apache/metron#1153 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7a8c2467 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7a8c2467 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7a8c2467 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: 7a8c246748a2c9e8c5b9230800b075dd99a7f3a4 Parents: 73dc63e Author: merrimanr Authored: Fri Aug 10 16:46:31 2018 -0500 Committer: rmerriman Committed: Fri Aug 10 16:46:31 2018 -0500 -- .../CURRENT/configuration/metron-rest-env.xml | 9 ++ .../package/scripts/params/params_linux.py | 1 + .../METRON/CURRENT/package/templates/metron.j2 | 1 + .../METRON/CURRENT/themes/metron_theme.json | 10 ++ metron-interface/metron-rest/README.md | 2 ++ .../src/main/config/rest_application.yml| 1 + .../apache/metron/rest/MetronRestConstants.java | 1 + .../metron/rest/config/PcapJobSupplier.java | 2 +- .../rest/service/impl/PcapServiceImpl.java | 12 ++- .../apache/metron/rest/mock/MockPcapJob.java| 8 + .../rest/service/impl/PcapServiceImplTest.java | 7 - metron-platform/metron-pcap-backend/README.md | 2 ++ .../org/apache/metron/pcap/query/CliParser.java | 4 +++ .../org/apache/metron/pcap/query/PcapCli.java | 3 ++ .../apache/metron/pcap/query/PcapCliTest.java | 33 ++-- .../apache/metron/pcap/config/PcapConfig.java | 10 ++ 16 files changed, 101 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/7a8c2467/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml index 20f9767..895c091 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml @@ -192,5 +192,14 @@ The number of pcaps written to a page/file as a result of a pcap query. 10 + +pcap_yarn_queue +Pcap YARN Queue +The YARN queue pcap jobs will be submitted to. + + +true + + http://git-wip-us.apache.org/repos/asf/metron/blob/7a8c2467/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 73d3469..4f8a9a7 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -390,6 +390,7 @@ pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path'] pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pcap_base_interim_result_path'] pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path'] pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size'] +pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue'] pcap_configured_flag_file = status_params.pcap_configured_flag_file # MapReduce http://git-wip-us.apache.org/repos/asf/metron/blob/7a8c2467/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 index 278d
metron git commit: METRON-1702 Reload a running job in the UI (merrimanr) closes apache/metron#1156
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1554-pcap-query-panel b29833cc5 -> 05316a4d9
METRON-1702 Reload a running job in the UI (merrimanr) closes apache/metron#1156
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/05316a4d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/05316a4d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/05316a4d
Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 05316a4d989d7546058e73b3a75eef435f2608a0
Parents: b29833c
Author: merrimanr
Authored: Fri Aug 10 08:27:06 2018 -0500
Committer: rmerriman
Committed: Fri Aug 10 08:27:06 2018 -0500
--
.../src/app/pcap/model/pcap.request.ts | 4 +-
.../pcap-filters/pcap-filters.component.spec.ts | 60 +-
.../pcap/pcap-filters/pcap-filters.component.ts | 26 --
.../pcap/pcap-panel/pcap-panel.component.html | 2 +-
.../pcap-panel/pcap-panel.component.spec.ts | 37 +
.../app/pcap/pcap-panel/pcap-panel.component.ts | 87 +---
.../src/app/pcap/service/pcap.service.spec.ts | 44 ++
.../src/app/pcap/service/pcap.service.ts| 15
.../rest/service/impl/PcapServiceImpl.java | 10 +--
.../rest/service/impl/PcapServiceImplTest.java | 36
10 files changed, 254 insertions(+), 67 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/05316a4d/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
index d00a6ac..8afc963 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
@@ -20,9 +20,9 @@ export class PcapRequest {
startTimeMs: number = 0;
endTimeMs: number = 15;
ipSrcAddr: string = '';
- ipSrcPort: number = 0;
+ ipSrcPort: number;
ipDstAddr: string = '';
- ipDstPort: number = 0;
+ ipDstPort: number;
protocol: string = '';
packetFilter: string = '';
includeReverse: boolean = false;
http://git-wip-us.apache.org/repos/asf/metron/blob/05316a4d/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
index 4336b22..c1d8664 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
@@ -21,7 +21,7 @@ import { By } from '@angular/platform-browser';
import { PcapFiltersComponent } from './pcap-filters.component';
import { FormsModule } from '../../../../node_modules/@angular/forms';
-import { Component, Input, Output, EventEmitter, DebugElement } from
'@angular/core';
+import { Component, Input, Output, EventEmitter, DebugElement, SimpleChange }
from '@angular/core';
import { PcapRequest } from '../model/pcap.request';
@Component({
@@ -171,6 +171,14 @@ describe('PcapFiltersComponent', () => {
component.onSubmit();
});
+ it('Port fields should be missing by default', () => {
+component.search.emit = (model: PcapRequest) => {
+ expect(model.ipSrcPort).toBeFalsy();
+ expect(model.ipDstPort).toBeFalsy();
+};
+component.onSubmit();
+ });
+
it('Filter should have an output called search', () => {
component.search.subscribe((filterModel) => {
expect(filterModel).toBeDefined();
@@ -194,14 +202,60 @@ describe('PcapFiltersComponent', () => {
expect(fixture.componentInstance.model.hasOwnProperty('startTimeMs')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('endTimeMs')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('ipSrcAddr')).toBeTruthy();
-
expect(fixture.componentInstance.model.hasOwnProperty('ipSrcPort')).toBeTruthy();
+
expect(fixture.componentInstance.model.hasOwnProperty('ipSrcPort')).toBeFalsy();
expect(fixture.componentInstance.model.hasOwnProperty('ipDstAddr')).toBeTruthy();
-
expect(fixture.componentInstance.model.hasOwnProperty('ipDstPort')).toBeTruthy();
+
expect(fixture.componentInstance.model.hasOwnProperty('ipDstPort')).toBeFalsy();
expect(fixture.componentInstance.model.hasOwnProperty('protocol')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('packetFilter')).toBeT
