metron git commit: METRON-1397 Support for JSON Path and complex documents in JSONMapParser closes apache/incubator-metron#914

2018-03-15 Thread cestella 
Repository: metron
Updated Branches:
  refs/heads/master 85d12475c -> 9c5d9d766


METRON-1397 Support for JSON Path and complex documents in JSONMapParser closes 
apache/incubator-metron#914


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9c5d9d76
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9c5d9d76
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9c5d9d76

Branch: refs/heads/master
Commit: 9c5d9d76644fc07bae36644906f52e0422f33d0e
Parents: 85d1247
Author: ottobackwards 
Authored: Thu Mar 15 14:17:31 2018 -0400
Committer: cstella 
Committed: Thu Mar 15 14:17:31 2018 -0400

--
 dependencies_with_url.csv   |   3 +
 .../docker/rpm-docker/SPECS/metron.spec |   1 +
 .../jsonMapQuery/parsed/jsonMapExampleParsed|   2 +
 .../data/jsonMapQuery/raw/jsonMapExampleOutput  |   1 +
 metron-platform/metron-parsers/README.md|  12 ++
 metron-platform/metron-parsers/pom.xml  |   5 +
 .../config/zookeeper/parsers/jsonMapQuery.json  |   5 +
 .../metron/parsers/json/JSONMapParser.java  | 145 +
 .../JSONMapQueryIntegrationTest.java|  36 
 .../validation/SampleDataValidation.java|   2 +-
 .../parsers/json/JSONMapParserQueryTest.java| 201 +++
 .../metron/test/utils/ValidationUtils.java  |  46 -
 12 files changed, 406 insertions(+), 53 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/dependencies_with_url.csv
--
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index e2b947b..1e73eb1 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -22,6 +22,9 @@ com.flipkart.zjsonpatch:zjsonpatch:jar:0.3.4:compile,Apache 
v2, https://github.c
 com.google.protobuf:protobuf-java:jar:2.5.0:compile,New BSD 
license,http://code.google.com/p/protobuf
 com.google.protobuf:protobuf-java:jar:2.6.1:compile,New BSD 
license,http://code.google.com/p/protobuf
 com.jcraft:jsch:jar:0.1.42:compile,BSD,http://www.jcraft.com/jsch/
+com.jayway.jsonpath:json-path:jar:2.3.0:compile,Apache 
v2,https://github.com/json-path/JsonPath
+net.minidev:accessors-smart:jar:1.2:compile,Apache 
v2,https://github.com/netplex/json-smart-v2
+net.minidev:json-smart:jar:2.3:compile,Apache 
v2,https://github.com/netplex/json-smart-v2
 com.maxmind.db:maxmind-db:jar:1.2.1:compile,CC-BY-SA 
3.0,https://github.com/maxmind/MaxMind-DB
 com.maxmind.geoip2:geoip2:jar:2.8.0:compile,Apache 
v2,https://github.com/maxmind/GeoIP2-java
 com.sun.xml.bind:jaxb-impl:jar:2.2.3-1:compile,CDDL,http://jaxb.java.net/

http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec 
b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 265d595..cc01d7c 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -147,6 +147,7 @@ This package installs the Metron Parser files
 %{metron_home}/bin/start_parser_topology.sh
 %{metron_home}/config/zookeeper/parsers/bro.json
 %{metron_home}/config/zookeeper/parsers/jsonMap.json
+%{metron_home}/config/zookeeper/parsers/jsonMapQuery.json
 %{metron_home}/config/zookeeper/parsers/snort.json
 %{metron_home}/config/zookeeper/parsers/squid.json
 %{metron_home}/config/zookeeper/parsers/websphere.json

http://git-wip-us.apache.org/repos/asf/metron/blob/9c5d9d76/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed
--
diff --git 
a/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed
 
b/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed
new file mode 100644
index 000..e614bda
--- /dev/null
+++ 
b/metron-platform/metron-integration-test/src/main/sample/data/jsonMapQuery/parsed/jsonMapExampleParsed
@@ -0,0 +1,2 @@
+{ "string" : "bar", "number" : 2, "ignored" : [ "blah" ], "original_string":"{ 
\"string\" : \"bar\", \"number\" : 2, \"ignored\" : [ \"blah\" ] 
}","timestamp":1, 
"source.type":"jsonMapQuery","guid":"this-is-random-uuid-will-be-36-chars" }
+{ "number" : 7 , "original_string" : "{ \"number\" : 7 }", 
"source.type":"jsonMapQuery","timestamp":1,"guid":"this-is-random-uuid-will-be-36-chars"}

[2/2] metron git commit: METRON-1299 In MetronError tests, don't test for HostName if getHostName wouldn't work closes apache/incubator-metron#924

2018-03-15 Thread cestella 
METRON-1299 In MetronError tests, don't test for HostName if getHostName 
wouldn't work closes apache/incubator-metron#924


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/85d12475
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/85d12475
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/85d12475

Branch: refs/heads/master
Commit: 85d12475cf363f0b9d6bdd973ec9860550f73980
Parents: e69ce21
Author: ottobackwards 
Authored: Thu Mar 15 14:13:44 2018 -0400
Committer: cstella 
Committed: Thu Mar 15 14:13:44 2018 -0400

--
 .../metron/common/error/MetronErrorTest.java  | 18 +++---
 1 file changed, 15 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/85d12475/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java
--
diff --git 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java
 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java
index 5e505a8..e7390de 100644
--- 
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java
+++ 
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/error/MetronErrorTest.java
@@ -18,15 +18,16 @@
 package org.apache.metron.common.error;
 
 import com.google.common.collect.Sets;
-import com.google.common.primitives.Bytes;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.metron.common.Constants;
 import org.json.simple.JSONObject;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.Arrays;
 
-import static org.apache.metron.common.Constants.ErrorFields.RAW_MESSAGE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -53,7 +54,18 @@ public class MetronErrorTest {
 assertEquals(Constants.ErrorType.PARSER_ERROR.getType(), 
errorJSON.get(Constants.ErrorFields.ERROR_TYPE.getName()));
 assertEquals("error", errorJSON.get(Constants.SENSOR_TYPE));
 assertEquals("sensorType", 
errorJSON.get(Constants.ErrorFields.FAILED_SENSOR_TYPE.getName()));
-assertTrue(((String) 
errorJSON.get(Constants.ErrorFields.HOSTNAME.getName())).length() > 0);
+
+String hostName = null;
+try {
+  hostName = InetAddress.getLocalHost().getHostName();
+} catch (UnknownHostException uhe) {
+  // unable to get the hostname on this machine, don't test it
+}
+
+if (!StringUtils.isEmpty(hostName)) {
+  assertTrue(((String) 
errorJSON.get(Constants.ErrorFields.HOSTNAME.getName())).length() > 0);
+  assertEquals(hostName, (String) 
errorJSON.get(Constants.ErrorFields.HOSTNAME.getName()));
+}
 assertTrue(((long) 
errorJSON.get(Constants.ErrorFields.TIMESTAMP.getName())) > 0);
   }

[1/2] metron git commit: METRON-1485 Upgrade vagrant for dev environments closes apache/incubator-metron#959

2018-03-15 Thread cestella 
Repository: metron
Updated Branches:
  refs/heads/master 26c5d3065 -> 85d12475c


METRON-1485 Upgrade vagrant for dev environments closes 
apache/incubator-metron#959


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e69ce213
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e69ce213
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e69ce213

Branch: refs/heads/master
Commit: e69ce213aaad84fd49d972c329a4bf4bf7d4b2ad
Parents: 26c5d30
Author: JonZeolla 
Authored: Thu Mar 15 14:09:28 2018 -0400
Committer: cstella 
Committed: Thu Mar 15 14:09:28 2018 -0400

--
 metron-deployment/development/centos6/README.md  | 2 +-
 metron-deployment/development/ubuntu14/README.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/e69ce213/metron-deployment/development/centos6/README.md
--
diff --git a/metron-deployment/development/centos6/README.md 
b/metron-deployment/development/centos6/README.md
index 5132c30..bd8553c 100644
--- a/metron-deployment/development/centos6/README.md
+++ b/metron-deployment/development/centos6/README.md
@@ -31,7 +31,7 @@ The computer used to deploy Apache Metron will need to have 
the following compon
 
  - [Ansible](https://github.com/ansible/ansible) (2.0.0.2 or 2.2.2.0)
  - [Docker](https://www.docker.com/community-edition)
- - [Vagrant](https://www.vagrantup.com) 1.8+
+ - [Vagrant](https://www.vagrantup.com) 2.0+
  - [Vagrant Hostmanager 
Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager)
  - [Virtualbox](https://virtualbox.org) 5.0+
  - Python 2.7

http://git-wip-us.apache.org/repos/asf/metron/blob/e69ce213/metron-deployment/development/ubuntu14/README.md
--
diff --git a/metron-deployment/development/ubuntu14/README.md 
b/metron-deployment/development/ubuntu14/README.md
index af7d3a3..5856911 100644
--- a/metron-deployment/development/ubuntu14/README.md
+++ b/metron-deployment/development/ubuntu14/README.md
@@ -31,7 +31,7 @@ The computer used to deploy Apache Metron will need to have 
the following compon
 
  - [Ansible](https://github.com/ansible/ansible) (2.0.0.2 or 2.2.2.0)
  - [Docker](https://www.docker.com/community-edition)
- - [Vagrant](https://www.vagrantup.com) 1.8+
+ - [Vagrant](https://www.vagrantup.com) 2.0+
  - [Vagrant Hostmanager 
Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager)
  - [Virtualbox](https://virtualbox.org) 5.0+
  - Python 2.7

metron git commit: METRON-1490: Better error message when user specifies an enrichment type that doesn't exist closes apache/metron#963

2018-03-15 Thread cestella 
Repository: metron
Updated Branches:
  refs/heads/master 03a4aa384 -> 37662d3bd


METRON-1490: Better error message when user specifies an enrichment type that 
doesn't exist closes apache/metron#963


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/37662d3b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/37662d3b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/37662d3b

Branch: refs/heads/master
Commit: 37662d3bd8a4162bef4474be7ab60a2fabbe3e2f
Parents: 03a4aa3
Author: cstella 
Authored: Thu Mar 15 12:05:39 2018 -0400
Committer: cstella 
Committed: Thu Mar 15 12:05:39 2018 -0400

--
 .../enrichment/parallel/ParallelEnricher.java   |  5 +++
 .../parallel/ParallelEnricherTest.java  | 39 +++-
 2 files changed, 43 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/metron/blob/37662d3b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
--
diff --git 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
index 2238c92..6ddb892 100644
--- 
a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
+++ 
b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/parallel/ParallelEnricher.java
@@ -18,6 +18,7 @@
 package org.apache.metron.enrichment.parallel;
 
 import com.github.benmanes.caffeine.cache.stats.CacheStats;
+import com.google.common.base.Joiner;
 import org.apache.metron.common.Constants;
 import 
org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler;
@@ -152,6 +153,10 @@ public class ParallelEnricher {
 for(Map.Entry task : tasks.entrySet()) {
   //task is the list of enrichment tasks for the task.getKey() adapter
   EnrichmentAdapter adapter = 
enrichmentsByType.get(task.getKey());
+  if(adapter == null) {
+throw new IllegalStateException("Unable to find an adapter for " + 
task.getKey()
++ ", possible adapters are: " + 
Joiner.on(",").join(enrichmentsByType.keySet()));
+  }
   for(JSONObject m : task.getValue()) {
 /* now for each unit of work (each of these only has one element in 
them)
  * the key is the field name and the value is value associated with 
that field.

http://git-wip-us.apache.org/repos/asf/metron/blob/37662d3b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
--
diff --git 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
index c3a3109..4a4573b 100644
--- 
a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
+++ 
b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/parallel/ParallelEnricherTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.metron.enrichment.parallel;
 
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableMap;
 import org.adrianwalker.multilinestring.Multiline;
 import org.apache.metron.common.Constants;
@@ -24,6 +25,7 @@ import 
org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig;
 import org.apache.metron.common.utils.JSONUtils;
 import org.apache.metron.enrichment.adapters.stellar.StellarAdapter;
 import org.apache.metron.enrichment.bolt.CacheKey;
+import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
 import org.apache.metron.stellar.dsl.Context;
 import org.apache.metron.stellar.dsl.StellarFunctions;
 import org.json.simple.JSONObject;
@@ -32,6 +34,7 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class ParallelEnricherTest {
@@ -61,6 +64,7 @@ public class ParallelEnricherTest {
   private static ParallelEnricher enricher;
   private static Context stellarContext;
   private static AtomicInteger numAccesses = new AtomicInteger(0);
+  private static Map enrichmentsByType;
   @BeforeClass
   public static void setup() {
 ConcurrencyContext infrastructure = new ConcurrencyContext();
@@ -75,7 +79,8 @@ public class