Repository: metron Updated Branches: refs/heads/master f523c1795 -> de2c871ab
METRON-995: Temporary variables in stellar enrichments which are maps do not function as expected closes apache/metron#615 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/de2c871a Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/de2c871a Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/de2c871a Branch: refs/heads/master Commit: de2c871abf3e259311281743952eb465362cf90c Parents: f523c17 Author: cstella <[email protected]> Authored: Mon Jun 19 10:37:50 2017 -0400 Committer: cstella <[email protected]> Committed: Mon Jun 19 10:37:50 2017 -0400 ---------------------------------------------------------------------- .../enrichment/handler/StellarConfig.java | 13 ++--- .../metron/common/stellar/StellarCompiler.java | 2 + .../StellarEnrichmentConfigTest.java | 37 +++++++++++++ .../configuration/StellarEnrichmentTest.java | 44 +++++++++++++++- .../metron/common/stellar/StellarTest.java | 5 ++ .../adapters/stellar/StellarAdapter.java | 35 ++++++++++--- .../adapters/stellar/StellarAdapterTest.java | 55 ++++++++++++++++++++ .../integration/EnrichmentIntegrationTest.java | 5 ++ .../main/config/zookeeper/enrichments/test.json | 4 +- 9 files changed, 185 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java index 7d49fbd..a888b81 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java @@ -17,18 +17,18 @@ */ package org.apache.metron.common.configuration.enrichment.handler; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; +import com.google.common.base.Joiner; import org.apache.metron.common.stellar.StellarAssignment; import org.apache.metron.common.stellar.StellarProcessor; import org.json.simple.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.*; import java.util.function.Function; public class StellarConfig implements Config { - + protected static final Logger _LOG = LoggerFactory.getLogger(StellarConfig.class); @Override public List<String> getSubgroups(Iterable<Map.Entry<String, Object>> config) { boolean includeEmpty = false; @@ -81,10 +81,10 @@ public class StellarConfig implements Config { { StellarProcessor processor = new StellarProcessor(); List<JSONObject> messages = new ArrayList<>(); - Map<String, String> defaultStellarStatementGroup = new HashMap<>(); + List<String> defaultStellarStatementGroup = new ArrayList<>(); for(Map.Entry<String, Object> kv : config) { if(kv.getValue() instanceof String) { - defaultStellarStatementGroup.put(kv.getKey(), (String)kv.getValue()); + defaultStellarStatementGroup.add((String)kv.getValue()); } else if(kv.getValue() instanceof Map) { JSONObject ret = new JSONObject(); @@ -103,6 +103,7 @@ public class StellarConfig implements Config { ret.put("", getMessage(getFields(processor, defaultStellarStatementGroup), message)); messages.add(ret); } + _LOG.debug("Stellar enrichment split: {}", messages ); return messages; } http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java index 136728d..f4e101b 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/stellar/StellarCompiler.java @@ -17,6 +17,7 @@ */ package org.apache.metron.common.stellar; +import org.antlr.v4.runtime.ParserRuleContext; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.metron.common.dsl.Context; import org.apache.metron.common.dsl.Token; @@ -347,6 +348,7 @@ public class StellarCompiler extends StellarBaseListener { }, DeferredFunction.class, context)); } + @Override public void exitVariable(StellarParser.VariableContext ctx) { final FrameContext.Context context = getArgContext(); http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java index 0883672..32fb00e 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentConfigTest.java @@ -18,7 +18,9 @@ package org.apache.metron.common.configuration; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; +import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler; import org.apache.metron.common.configuration.enrichment.handler.Configs; @@ -33,6 +35,41 @@ import java.util.Map; public class StellarEnrichmentConfigTest extends StellarEnrichmentTest { + /** + { + "fieldMap": { + "stellar" : { + "config" : [ + "dga_model_endpoint := MAAS_GET_ENDPOINT('dga')", + "dga_result_map := MAAS_MODEL_APPLY( dga_model_endpoint, { 'host' : domain_without_subdomains } )", + "dga_result := MAP_GET('is_malicious', dga_result_map)", + "is_dga := dga_result != null && dga_result == 'dga'", + "dga_model_version := MAP_GET('version', dga_model_endpoint)", + "dga_model_endpoint := null", + "dga_result_map := null", + "dga_result := null" + ] + } + } + } + */ + @Multiline + public static String conf; + + @Test + public void testSplitter_listWithTemporaryVariables() throws IOException { + JSONObject message = new JSONObject(ImmutableMap.of("domain_without_subdomains", "yahoo.com")); + EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(conf, EnrichmentConfig.class); + Assert.assertNotNull(enrichmentConfig.getEnrichmentConfigs().get("stellar")); + ConfigHandler handler = enrichmentConfig.getEnrichmentConfigs().get("stellar"); + List<JSONObject> splits = Configs.STELLAR.splitByFields(message, null, x -> null, handler ); + Assert.assertEquals(1, splits.size()); + Map<String, Object> split = (Map<String, Object>)(splits.get(0)).get(""); + Assert.assertEquals("yahoo.com", split.get("domain_without_subdomains")); + Assert.assertTrue(split.containsKey("dga_result")); + Assert.assertTrue(split.containsKey("dga_model_endpoint")); + Assert.assertTrue(split.containsKey("dga_result_map")); + } @Test public void testSplitter_default() throws IOException { http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java index 3af15cd..8eb9bbc 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java @@ -59,7 +59,25 @@ public class StellarEnrichmentTest { */ @Multiline public static String defaultStellarConfig_list; - public static List<String> DEFAULT_CONFIGS = ImmutableList.of(defaultStellarConfig_list, defaultStellarConfig_map); + + /** + { + "fieldMap": { + "stellar" : { + "config" : [ + "stmt1 := TO_UPPER(source.type)", + "stmt4 := TO_LOWER(string)", + "stmt2 := TO_LOWER(stmt1)", + "stmt3 := TO_LOWER(string)", + "stmt4 := null" + ] + } + } + } + */ + @Multiline + public static String defaultStellarConfig_listWithTemp; + public static List<String> DEFAULT_CONFIGS = ImmutableList.of(defaultStellarConfig_list, defaultStellarConfig_map, defaultStellarConfig_listWithTemp); /** { @@ -100,7 +118,29 @@ public class StellarEnrichmentTest { */ @Multiline public static String groupedStellarConfig_list; - public static List<String> GROUPED_CONFIGS = ImmutableList.of(groupedStellarConfig_list, groupedStellarConfig_map); + + /** + { + "fieldMap": { + "stellar" : { + "config" : { + "group1" : [ + "stmt1 := TO_UPPER(source.type)", + "stmt2 := TO_LOWER(stmt1)" + ], + "group2" : [ + "stmt3 := TO_LOWER(string)", + "stmt4 := TO_LOWER(string)", + "stmt4 := null" + ] + } + } + } + } + */ + @Multiline + public static String groupedStellarConfig_listWithTemp; + public static List<String> GROUPED_CONFIGS = ImmutableList.of(groupedStellarConfig_listWithTemp, groupedStellarConfig_list, groupedStellarConfig_map); /** { http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java index 98abd57..d549e19 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/stellar/StellarTest.java @@ -183,8 +183,13 @@ public class StellarTest { Assert.assertEquals(ImmutableSet.of("one", "two") , processor.variablesUsed("if 1 < 2 then one else two")); } + { + Assert.assertEquals(ImmutableSet.of("bar") + , processor.variablesUsed("MAP_GET('foo', { 'foo' : bar})")); + } } + @Test public void testFunctionEmptyArgs() { { http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java index 53ef16f..4b78821 100644 --- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java +++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapter.java @@ -32,6 +32,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -101,7 +103,9 @@ public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable JSONObject ret = new JSONObject(); Iterable<Map.Entry<String, Object>> stellarStatements = getStellarStatements(handler, field); + _LOG.debug("message := {}", message); if(stellarStatements != null) { + List<String> mapEntries = new ArrayList<>(); for (Map.Entry<String, Object> kv : stellarStatements) { if(kv.getKey() != null && kv.getValue() != null) { if (kv.getValue() instanceof String) { @@ -114,14 +118,11 @@ public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable _PERF_LOG.debug("SLOW LOG: " + stellarStatement + " took" + duration + "ms"); } } + _LOG.debug("{} := {} yields {}", kv.getKey(), stellarStatement , o); if (o != null && o instanceof Map) { - for (Map.Entry<Object, Object> valueKv : ((Map<Object, Object>) o).entrySet()) { - String newKey = ((kv.getKey().length() > 0) ? kv.getKey() + "." : "") + valueKv.getKey(); - message.put(newKey, valueKv.getValue()); - ret.put(newKey, valueKv.getValue()); - } + mapEntries.add(kv.getKey()); } - else if(o == null) { + if(o == null) { message.remove(kv.getKey()); ret.remove(kv.getKey()); } @@ -132,6 +133,28 @@ public class StellarAdapter implements EnrichmentAdapter<CacheKey>,Serializable } } } + /* + We need to handle the map entries separately now. + We want to explode them out, so if "var" is + { + "foo" : "bar" + } + then we want "var.foo" == "bar" + and no "var" + */ + for(String mapEntry : mapEntries) { + String key = mapEntry; + Map<Object, Object> value = (Map<Object, Object>) ret.get(key); + if(value != null) { + _LOG.debug("Exploding map: {} == {}", key, value); + for (Map.Entry<Object, Object> valueKv : value.entrySet()) { + String newKey = ((key.length() > 0) ? key + "." : "") + valueKv.getKey(); + ret.put(newKey, valueKv.getValue()); + } + //removing the map from downstream + ret.remove(key); + } + } } return ret; } http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java index 5624d4b..724221d 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java @@ -18,6 +18,7 @@ package org.apache.metron.enrichment.adapters.stellar; import com.google.common.collect.ImmutableList; +import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.configuration.StellarEnrichmentTest; import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; import org.apache.metron.common.configuration.enrichment.handler.ConfigHandler; @@ -30,6 +31,8 @@ import org.json.simple.JSONObject; import org.junit.Assert; import org.junit.Test; +import java.util.List; + public class StellarAdapterTest extends StellarEnrichmentTest { StellarProcessor processor = new StellarProcessor(); @@ -131,4 +134,56 @@ public class StellarAdapterTest extends StellarEnrichmentTest { Assert.assertEquals(2, enriched.size()); } } + + /** + { + "fieldMap": { + "stellar" : { + "config" : { + "group1" : [ + "stmt1 := TO_UPPER(source.type)", + "stmt2 := { 'foo' : source.type }" + ] + } + } + } + } + */ + @Multiline + public static String mapConfig_subgroup; + /** + { + "fieldMap": { + "stellar" : { + "config" : [ + "stmt1 := TO_UPPER(source.type)", + "stmt2 := { 'foo' : source.type }" + ] + } + } + } + */ + @Multiline + public static String mapConfig_default; + + private void testMapEnrichment(String config, String field) throws Exception { + JSONObject message = getMessage(); + EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(config, EnrichmentConfig.class); + Assert.assertNotNull(enrichmentConfig.getEnrichmentConfigs().get("stellar")); + ConfigHandler handler = enrichmentConfig.getEnrichmentConfigs().get("stellar"); + JSONObject enriched = enrich(message, field, handler); + Assert.assertEquals(2, enriched.size()); + Assert.assertEquals("stellar_test", enriched.get("stmt2.foo")); + Assert.assertEquals("stellar_test".toUpperCase(), enriched.get("stmt1")); + } + + @Test + public void testMapEnrichment_subgroup() throws Exception { + testMapEnrichment(mapConfig_subgroup, "group1"); + } + + @Test + public void testMapEnrichment_default() throws Exception { + testMapEnrichment(mapConfig_default, ""); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java index 9634805..2149cb9 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java @@ -260,6 +260,11 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest { Assert.assertNotNull(jsonDoc.get(DST_IP)); Assert.assertNotNull(jsonDoc.get("ALL_CAPS")); + Assert.assertNotNull(jsonDoc.get("map.blah")); + Assert.assertNull(jsonDoc.get("map")); + Assert.assertNotNull(jsonDoc.get("one")); + Assert.assertEquals(1, jsonDoc.get("one")); + Assert.assertEquals(1, jsonDoc.get("map.blah")); Assert.assertNotNull(jsonDoc.get("foo")); Assert.assertEquals("TEST", jsonDoc.get("ALL_CAPS")); Assert.assertNotNull(jsonDoc.get("bar")); http://git-wip-us.apache.org/repos/asf/metron/blob/de2c871a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json index 1037b69..bba6679 100644 --- a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json +++ b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json @@ -16,7 +16,9 @@ "stellar" : { "config" : { "numeric" : { - "foo": "1 + 1" + "map" : "{ 'blah' : 1}" + ,"one" : "MAP_GET('blah', map)" + ,"foo": "1 + 1" } ,"ALL_CAPS" : "TO_UPPER(source.type)" ,"src_enrichment" : {
