METRON-1568: Stellar should have a _ special variable which returns the message in map form closes apache/incubator-metron#1021
Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1b9828e6 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1b9828e6 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1b9828e6 Branch: refs/heads/feature/METRON-1554-pcap-query-panel Commit: 1b9828e6a68fd39fd308ebb80f1d79cd490ceaec Parents: 523c38c Author: cstella <ceste...@gmail.com> Authored: Thu Jun 7 10:54:05 2018 -0400 Committer: cstella <ceste...@gmail.com> Committed: Thu Jun 7 10:54:05 2018 -0400 ---------------------------------------------------------------------- .../enrichment/handler/StellarConfig.java | 11 +- .../StellarTransformationTest.java | 30 ++- .../adapters/stellar/StellarAdapterTest.java | 26 ++ .../integration/EnrichmentIntegrationTest.java | 2 + .../main/config/zookeeper/enrichments/test.json | 1 + .../metron/pcap/filter/PcapFieldResolver.java | 5 + metron-stellar/stellar-common/README.md | 1 + .../metron/stellar/common/utils/ConcatMap.java | 256 +++++++++++++++++++ .../common/utils/StellarProcessorUtils.java | 51 +++- .../metron/stellar/dsl/MapVariableResolver.java | 8 + .../metron/stellar/dsl/VariableResolver.java | 1 + .../dsl/functions/DataStructureFunctions.java | 4 + .../stellar/common/utils/ConcatMapTest.java | 83 ++++++ .../stellar/dsl/functions/BasicStellarTest.java | 18 ++ 14 files changed, 487 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java index 62e0263..9f96a22 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java @@ -19,6 +19,7 @@ package org.apache.metron.common.configuration.enrichment.handler; import org.apache.metron.stellar.common.StellarAssignment; import org.apache.metron.stellar.common.StellarProcessor; +import org.apache.metron.stellar.dsl.VariableResolver; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -142,8 +143,14 @@ public class StellarConfig implements Config { { Map<String, Object> messageSegment = new HashMap<>(); - for(String variable : stellarFields) { - messageSegment.put(variable, message.get(variable)); + if(stellarFields.contains(VariableResolver.ALL_FIELDS)) { + //we need to include all of the fields here. + messageSegment.putAll(message); + } + else { + for (String variable : stellarFields) { + messageSegment.put(variable, message.get(variable)); + } } return messageSegment; } http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java index fc91844..3b7c7bb 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java @@ -78,6 +78,32 @@ public class StellarTransformationTest { @Multiline public static String badConfig; + /** + { "fieldTransformations" : [ + { "transformation" : "STELLAR" + ,"output" : [ "new_field"] + ,"config" : { + "new_field" : "MAP_GET('source.type', _)" + } + } + ] + } + */ + @Multiline + public static String configAll; + + @Test + public void testConfigAll() throws Exception { + SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(configAll)); + JSONObject input = new JSONObject(); + input.put("source.type", "test"); + for (FieldTransformer handler : c.getFieldTransformations()) { + handler.transformAndUpdate(input, Context.EMPTY_CONTEXT()); + } + Assert.assertEquals(2, input.size()); + Assert.assertTrue(input.containsKey("new_field")); + Assert.assertEquals("test", input.get("new_field")); + } /** { "fieldTransformations" : [ { "transformation" : "STELLAR" @@ -92,8 +118,8 @@ public class StellarTransformationTest { ] } */ - @Multiline - public static String configRename; + @Multiline + public static String configRename; @Test public void testStellarRename() throws Exception { http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java index cd1006c..e03e69f 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/adapters/stellar/StellarAdapterTest.java @@ -165,6 +165,7 @@ public class StellarAdapterTest extends StellarEnrichmentTest { @Multiline public static String mapConfig_default; + private void testMapEnrichment(String config, String field) throws Exception { JSONObject message = getMessage(); EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(config, EnrichmentConfig.class); @@ -185,4 +186,29 @@ public class StellarAdapterTest extends StellarEnrichmentTest { public void testMapEnrichment_default() throws Exception { testMapEnrichment(mapConfig_default, ""); } + + /** + { + "fieldMap": { + "stellar" : { + "config" : [ + "stmt1 := MAP_GET('source.type', _)" + ] + } + } + } + */ + @Multiline + public static String allVariableConfig; + + @Test + public void testAllVariableUsage() throws Exception { + JSONObject message = getMessage(); + EnrichmentConfig enrichmentConfig = JSONUtils.INSTANCE.load(allVariableConfig, EnrichmentConfig.class); + Assert.assertNotNull(enrichmentConfig.getEnrichmentConfigs().get("stellar")); + ConfigHandler handler = enrichmentConfig.getEnrichmentConfigs().get("stellar"); + JSONObject enriched = enrich(message, "", handler); + Assert.assertEquals("stellar_test", enriched.get("stmt1")); + } + } http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java index 2e22eab..188f18b 100644 --- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java +++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java @@ -294,6 +294,8 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest { Assert.assertEquals(1, jsonDoc.get("one")); Assert.assertEquals(1, jsonDoc.get("map.blah")); Assert.assertNotNull(jsonDoc.get("foo")); + Assert.assertNotNull(jsonDoc.get("alt_src_type")); + Assert.assertEquals("test", jsonDoc.get("alt_src_type")); Assert.assertEquals("TEST", jsonDoc.get("ALL_CAPS")); Assert.assertNotNull(jsonDoc.get("bar")); Assert.assertEquals("TEST", jsonDoc.get("bar")); http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json index 4130943..9b997a6 100644 --- a/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json +++ b/metron-platform/metron-integration-test/src/main/config/zookeeper/enrichments/test.json @@ -19,6 +19,7 @@ "map" : "{ 'blah' : 1}" ,"one" : "MAP_GET('blah', map)" ,"foo": "1 + 1" + ,"alt_src_type" : "MAP_GET('source.type', _)" } ,"ALL_CAPS" : "TO_UPPER(source.type)" ,"src_enrichment" : { http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java index 4ea6836..e3ac7e5 100644 --- a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java +++ b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/filter/PcapFieldResolver.java @@ -18,6 +18,8 @@ package org.apache.metron.pcap.filter; +import com.google.common.collect.ImmutableList; +import org.apache.metron.stellar.common.utils.ConcatMap; import org.apache.metron.stellar.dsl.VariableResolver; import java.util.HashMap; @@ -32,6 +34,9 @@ public class PcapFieldResolver implements VariableResolver { @Override public Object resolve(String variable) { + if(variable.equals(VariableResolver.ALL_FIELDS)) { + return new ConcatMap(ImmutableList.of(fieldsMap)); + } return fieldsMap.get(variable); } http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/README.md ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md index 291f1be..dcf0dbf 100644 --- a/metron-stellar/stellar-common/README.md +++ b/metron-stellar/stellar-common/README.md @@ -37,6 +37,7 @@ For a variety of components (threat intelligence triage and field transformation The Stellar language supports the following: * Referencing fields in the enriched JSON +* Referencing all fields in the enriched JSON via the `_` reserved variable name. * String literals are quoted with either `'` or `"` * String literals support escaping for `'`, `"`, `\t`, `\r`, `\n`, and backslash * The literal `'\'foo\''` would represent `'foo'` http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java new file mode 100644 index 0000000..53a52a6 --- /dev/null +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/ConcatMap.java @@ -0,0 +1,256 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.metron.stellar.common.utils; + + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.KryoSerializable; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.google.common.base.Joiner; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * ConcatMap is a lazy concatenation of a list of Maps. It is lazy in that it does not construct + * a union of all of the maps, but rather keeps the maps separate. Key/Value resolution is + * done via a first-wins strategy (i.e. the first map which has a key will be used). + * + * Also, note, that this is an immutable map, so operations which require mutation will have + * UnsupportedOperationException thrown. + */ +public class ConcatMap implements Map<String, Object>, Serializable, KryoSerializable { + List<Map> variableMappings = new ArrayList<>(); + public ConcatMap(List<Map> variableMappings) { + this.variableMappings = variableMappings; + } + + @Override + public int size() { + int size = 0; + for(Map m : variableMappings) { + size += m.size(); + } + return size; + } + + @Override + public boolean isEmpty() { + boolean isEmpty = true; + for(Map m : variableMappings) { + isEmpty &= m.isEmpty(); + } + return isEmpty; + } + + /** + * If any maps contains the key, then this will return true. + * @param key + * @return + */ + @Override + public boolean containsKey(Object key) { + for(Map m : variableMappings) { + if(m.containsKey(key)) { + return true; + } + } + return false; + } + + /** + * + * If any maps contains the value, then this will return true. + * @param value + * @return + */ + @Override + public boolean containsValue(Object value) { + for(Map m : variableMappings) { + if(m.containsValue(value)) { + return true; + } + } + return false; + } + + /** + * The first map which contains the key will have the associated value returned. + * @param key + * @return + */ + @Override + public Object get(Object key) { + Object ret = null; + for(Map m : variableMappings) { + ret = m.get(key); + if(ret != null) { + break; + } + } + return ret; + } + + /** + * This is an immutable map and this operation is not supported. + * @param key + * @param value + * @return + */ + @Override + public Object put(String key, Object value) { + throw new UnsupportedOperationException("Merged map is immutable."); + } + + /** + * + * This is an immutable map and this operation is not supported. + * @param key + * @return + */ + @Override + public Object remove(Object key) { + throw new UnsupportedOperationException("Merged map is immutable."); + } + + /** + * + * This is an immutable map and this operation is not supported. + * @param m + */ + @Override + public void putAll(Map<? extends String, ?> m) { + throw new UnsupportedOperationException("Merged map is immutable."); + } + + /** + * + * This is an immutable map and this operation is not supported. + */ + @Override + public void clear() { + throw new UnsupportedOperationException("Merged map is immutable."); + } + + @Override + public Set<String> keySet() { + Set<String> ret = null; + for(Map m : variableMappings) { + if(ret == null) { + ret = m.keySet(); + } + else { + ret = Sets.union(ret, m.keySet()); + } + } + return ret; + } + + /** + * Note: this makes a copy of the values, so it is not fundamentally lazy. + * @return + */ + @Override + public Collection<Object> values() { + Collection<Object> ret = new ArrayList<>(size()); + for(Map m : variableMappings) { + ret.addAll(m.values()); + } + return ret; + } + + /** + * This is a lazy entry collection of the associated maps. If there are duplicate keys, they will appear + * twice here, so be careful. + * @return + */ + @Override + public Set<Entry<String, Object>> entrySet() { + Set<Entry<String, Object>> ret = null; + for(Map m : variableMappings) { + if(ret == null) { + ret = m.entrySet(); + } + else { + ret = Sets.union(ret, m.entrySet()); + } + } + return ret; + } + + @Override + public String toString() { + Iterable<Iterable<Map.Entry<Object, Object>>> transformed = + Iterables.transform(variableMappings, x -> x.entrySet()); + Iterable<Map.Entry<Object, Object>> it = Iterables.filter( Iterables.concat(transformed) + , x -> x.getValue() != null + ); + return "{" + Joiner.on(", ").join(it) + "}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ConcatMap concatMap = (ConcatMap) o; + + return variableMappings != null ? variableMappings.equals(concatMap.variableMappings) : concatMap.variableMappings == null; + + } + + @Override + public int hashCode() { + return variableMappings != null ? variableMappings.hashCode() : 0; + } + + @Override + public void write(Kryo kryo, Output output) { + int numVariableMappings = variableMappings.isEmpty()?0:variableMappings.size(); + output.writeShort(numVariableMappings); + for(Map m : variableMappings) { + byte[] b = m == null?new byte[]{}:SerDeUtils.toBytes(m); + output.writeInt(b.length); + if(b.length > 0) { + output.writeBytes(b); + } + } + } + + @Override + public void read(Kryo kryo, Input input) { + int numVariableMappings = input.readShort(); + variableMappings = new ArrayList<>(numVariableMappings); + for(int i = 0;i < numVariableMappings;++i) { + int size = input.readInt(); + if(size > 0) { + byte[] bytes = input.readBytes(size); + Map m = SerDeUtils.fromBytes(bytes, Map.class); + variableMappings.add(m); + } + } + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java index 4ad5a40..51e7aaa 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/StellarProcessorUtils.java @@ -62,14 +62,14 @@ public class StellarProcessorUtils { * validates successfully and produces a result that can be serialized correctly. * * @param expression The expression to execute. - * @param variables The variables to expose to the expression. + * @param varResolver The variable resolver to use * @param context The execution context. * @return The result of executing the expression. */ - public static Object run(String expression, Map<String, Object> variables, Context context) { + public static Object run(String expression, VariableResolver varResolver, Context context) { validate(expression, context); - Object result = execute(expression, variables, context); + Object result = execute(expression, varResolver, context); ensureKryoSerializable(result, expression); ensureJavaSerializable(result, expression); @@ -77,19 +77,44 @@ public class StellarProcessorUtils { } /** + * Execute and validate a Stellar expression. + * + * <p>This is intended for use while unit testing Stellar expressions. This ensures that the expression + * validates successfully and produces a result that can be serialized correctly. + * + * @param expression The expression to execute. + * @param variables The variables to expose to the expression. + * @param context The execution context. + * @return The result of executing the expression. + */ + public static Object run(String expression, Map<String, Object> variables, Context context) { + VariableResolver varResolver = new DefaultVariableResolver( + x -> { + if(x.equals(MapVariableResolver.ALL_FIELDS)) { + return variables; + } + return variables.get(x); + } + ,x-> x.equals(MapVariableResolver.ALL_FIELDS) || variables.containsKey(x) + ); + return run(expression, varResolver, context); + } + + /** * Execute a Stellar expression. * * @param expression The expression to execute. - * @param variables The variables available to the expression. + * @param variableResolver Variable Resolver to use * @param context The execution context. * @return The result of executing the expression. */ - private static Object execute(String expression, Map<String, Object> variables, Context context) { + private static Object execute(String expression, VariableResolver variableResolver, Context context) { StellarProcessor processor = new StellarProcessor(); + Object result = processor.parse( expression, - new DefaultVariableResolver(x -> variables.get(x), x -> variables.containsKey(x)), + variableResolver, StellarFunctions.FUNCTION_RESOLVER(), context); @@ -180,6 +205,20 @@ public class StellarProcessorUtils { * validates successfully and produces a result that can be serialized correctly. * * @param expression The expression to execute. + * @param variables The variables to expose to the expression. + * @return The result of executing the expression. + */ + public static Object run(String expression, VariableResolver variables) { + return run(expression, variables, Context.EMPTY_CONTEXT()); + } + + /** + * Execute and validate a Stellar expression. + * + * <p>This is intended for use while unit testing Stellar expressions. This ensures that the expression + * validates successfully and produces a result that can be serialized correctly. + * + * @param expression The expression to execute. * @param context The execution context. * @return The result of executing the expression. */ http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java index 4c02d99..872211d 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/MapVariableResolver.java @@ -18,12 +18,16 @@ package org.apache.metron.stellar.dsl; + +import org.apache.metron.stellar.common.utils.ConcatMap; + import java.util.ArrayList; import java.util.List; import java.util.Map; public class MapVariableResolver implements VariableResolver { + List<Map> variableMappings = new ArrayList<>(); public MapVariableResolver(Map variableMappingOne, Map... variableMapping) { @@ -45,6 +49,10 @@ public class MapVariableResolver implements VariableResolver { @Override public Object resolve(String variable) { + if(variable != null && variable.equals(VariableResolver.ALL_FIELDS)) { + return new ConcatMap(variableMappings); + } + for (Map variableMapping : variableMappings) { Object o = variableMapping.get(variable); if (o != null) { http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java index f2624e9..fb95d27 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/VariableResolver.java @@ -20,6 +20,7 @@ package org.apache.metron.stellar.dsl; public interface VariableResolver { + public static final String ALL_FIELDS = "_"; Object resolve(String variable); boolean exists(String variable); } http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java index eb1db48..bebe27d 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/DataStructureFunctions.java @@ -27,6 +27,7 @@ import org.apache.metron.stellar.common.utils.SerDeUtils; import java.util.Collection; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; public class DataStructureFunctions { @@ -211,6 +212,9 @@ public class DataStructureFunctions { if(o instanceof Collection) { return ((Collection)o).size(); } + else if(o instanceof Map) { + return ((Map)o).size(); + } else if(o instanceof String) { String val = (String) list.get(0); return val == null || val.isEmpty() ? 0 : val.length(); http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/ConcatMapTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/ConcatMapTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/ConcatMapTest.java new file mode 100644 index 0000000..4c078e8 --- /dev/null +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/ConcatMapTest.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.stellar.common.utils; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ConcatMapTest { + + @Test + public void testToString() throws Exception { + Map<String, Object> v1 = new HashMap<>(); + v1.put("k1", "v1"); + Map<String, Object> v2 = new HashMap<>(); + v2.put("k2", "v2"); + v2.put("k3", null); + Map<String, Object> union = new HashMap<String, Object>() {{ + putAll(v1); + put("k2", "v2"); + }}; + ConcatMap c = create(v1, v2); + Assert.assertEquals(c.toString(), union.toString()); + } + + private ConcatMap create(Map... ms) { + List<Map> l = new ArrayList<>(); + for(Map m : ms) { + l.add(m); + } + return new ConcatMap(l); + } + + private void assertKryoserializable(ConcatMap c) { + byte[] serialized = SerDeUtils.toBytes(c); + ConcatMap deserialized = SerDeUtils.fromBytes(serialized, ConcatMap.class); + Assert.assertEquals(deserialized, c); + } + + @Test + public void testKryoSerialization() { + Map<String, Object> v1 = new HashMap<>(); + v1.put("k1", "v1"); + Map<String, Object> v2 = new HashMap<>(); + v2.put("k2", "v2"); + v2.put("k3", null); + { + //multi maps + ConcatMap c = create(v1, v2); + assertKryoserializable(c); + } + { + //single maps + ConcatMap c = create(v1); + assertKryoserializable(c); + } + { + //empty maps + ConcatMap c = create(); + assertKryoserializable(c); + } + } + +} http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java index aa4462a..ea859c5 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java @@ -25,9 +25,11 @@ import org.apache.commons.lang3.StringUtils; import org.apache.metron.stellar.common.StellarProcessor; import org.apache.metron.stellar.dsl.Context; import org.apache.metron.stellar.dsl.DefaultVariableResolver; +import org.apache.metron.stellar.dsl.MapVariableResolver; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.Stellar; import org.apache.metron.stellar.dsl.StellarFunction; +import org.apache.metron.stellar.dsl.VariableResolver; import org.apache.metron.stellar.dsl.functions.resolver.ClasspathFunctionResolver; import org.junit.Assert; import org.junit.Ignore; @@ -957,4 +959,20 @@ public class BasicStellarTest { thrown.expectMessage("The rule 'TO_UPPER(protocol)' does not return a boolean value."); runPredicate("TO_UPPER(protocol)", new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))); } + + @Test + public void all_fields_test() { + final Map<String, Object> varMap1 = new HashMap<String, Object>(); + varMap1.put("field1", "val1"); + final Map<String, Object> varMap2 = new HashMap<String, Object>(); + varMap2.put("field2", "val2"); + VariableResolver resolver = new MapVariableResolver(varMap1, varMap2); + Assert.assertTrue(runPredicate("MAP_GET('field1', _) == 'val1'", resolver)); + Assert.assertTrue(runPredicate("MAP_GET('field2', _) == 'val2'", resolver)); + Assert.assertTrue(runPredicate("LENGTH(_) == 2", resolver)); + Map<String, Object> ret = (Map<String, Object>) run("_", resolver); + Assert.assertEquals(2, ret.size()); + Assert.assertEquals("val1", ret.get("field1")); + Assert.assertEquals("val2", ret.get("field2")); + } }
