Repository: incubator-metron Updated Branches: refs/heads/master a12faeb0e -> 1f83fdcba
METRON-741: Stellar Field Transformations should execute all of the transformations, not just the output closes apache/incubator-metron#465 Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/1f83fdcb Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/1f83fdcb Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/1f83fdcb Branch: refs/heads/master Commit: 1f83fdcba7d04242be23fe7462f72267d476bc7e Parents: a12faeb Author: cstella <ceste...@gmail.com> Authored: Mon Feb 27 09:02:11 2017 -0500 Committer: cstella <ceste...@gmail.com> Committed: Mon Feb 27 09:02:11 2017 -0500 ---------------------------------------------------------------------- .../common/configuration/FieldTransformer.java | 9 ++-- .../metron/common/dsl/MapVariableResolver.java | 6 ++- .../transformation/FieldTransformation.java | 3 +- .../transformation/RemoveTransformation.java | 3 +- .../SimpleFieldTransformation.java | 3 +- .../transformation/StellarTransformation.java | 22 ++++++--- .../transformation/FieldTransformationTest.java | 3 +- .../StellarTransformationTest.java | 51 ++++++++++++++++---- 8 files changed, 71 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java index c5ca7a2..46f420e 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/FieldTransformer.java @@ -26,17 +26,14 @@ import org.apache.metron.common.field.transformation.FieldTransformations; import org.json.simple.JSONObject; import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class FieldTransformer implements Serializable { private List<String> input = new ArrayList<>(); private List<String> output; private FieldTransformation transformation; private String transformationName; - private Map<String, Object> config = new HashMap<>(); + private LinkedHashMap<String, Object> config = new LinkedHashMap<>(); private boolean initialized = false; public FieldTransformer() { } @@ -71,7 +68,7 @@ public class FieldTransformer implements Serializable { return config; } - public void setConfig(Map<String, Object> config) { + public void setConfig(LinkedHashMap<String, Object> config) { this.config = config; } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java index 8d56af2..f180e80 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/dsl/MapVariableResolver.java @@ -27,9 +27,11 @@ public class MapVariableResolver implements VariableResolver { List<Map> variableMappings = new ArrayList<>(); public MapVariableResolver(Map variableMappingOne, Map... variableMapping) { - variableMappings.add(variableMappingOne); + if(variableMappingOne != null) { + variableMappings.add(variableMappingOne); + } for(Map m : variableMapping) { - if(m != null && m.size() > 0) { + if(m != null) { this.variableMappings.add(m); } } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java index a19d165..bb33bf7 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/FieldTransformation.java @@ -21,13 +21,14 @@ package org.apache.metron.common.field.transformation; import org.apache.metron.common.dsl.Context; import java.io.Serializable; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; public interface FieldTransformation extends Serializable { Map<String, Object> map( Map<String, Object> input , List<String> outputField - , Map<String, Object> fieldMappingConfig + , LinkedHashMap<String, Object> fieldMappingConfig , Map<String, Object> sensorConfig , Context context ); http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java index 19aab1d..3b5ae5f 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/RemoveTransformation.java @@ -23,6 +23,7 @@ import org.apache.metron.common.dsl.functions.resolver.FunctionResolver; import org.apache.metron.common.stellar.StellarPredicateProcessor; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -66,7 +67,7 @@ public class RemoveTransformation implements FieldTransformation { @Override public Map<String, Object> map( Map<String, Object> input , final List<String> outputFields - , Map<String, Object> fieldMappingConfig + , LinkedHashMap<String, Object> fieldMappingConfig , Map<String, Object> sensorConfig , Context context ) { http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java index 71b830c..1f8efbd 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/SimpleFieldTransformation.java @@ -21,6 +21,7 @@ package org.apache.metron.common.field.transformation; import com.google.common.collect.Iterables; import org.apache.metron.common.dsl.Context; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -28,7 +29,7 @@ public abstract class SimpleFieldTransformation implements FieldTransformation { @Override public Map<String, Object> map (Map<String, Object> input , List<String> outputField - , Map<String, Object> fieldMappingConfig + , LinkedHashMap<String, Object> fieldMappingConfig , Map<String, Object> sensorConfig , Context context ) http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java index 2bbbc80..e2ef7c9 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/field/transformation/StellarTransformation.java @@ -24,29 +24,35 @@ import org.apache.metron.common.dsl.StellarFunctions; import org.apache.metron.common.dsl.VariableResolver; import org.apache.metron.common.stellar.StellarProcessor; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class StellarTransformation implements FieldTransformation { @Override public Map<String, Object> map( Map<String, Object> input , List<String> outputField - , Map<String, Object> fieldMappingConfig + , LinkedHashMap<String, Object> fieldMappingConfig , Map<String, Object> sensorConfig , Context context ) { Map<String, Object> ret = new HashMap<>(); - VariableResolver resolver = new MapVariableResolver(ret, input, sensorConfig); + Map<String, Object> intermediateVariables = new HashMap<>(); + Set<String> outputs = new HashSet<>(outputField); + VariableResolver resolver = new MapVariableResolver(ret, intermediateVariables, input, sensorConfig); StellarProcessor processor = new StellarProcessor(); - for(String oField : outputField) { - Object transformObj = fieldMappingConfig.get(oField); + for(Map.Entry<String, Object> kv : fieldMappingConfig.entrySet()) { + String oField = kv.getKey(); + Object transformObj = kv.getValue(); if(transformObj != null) { try { Object o = processor.parse(transformObj.toString(), resolver, StellarFunctions.FUNCTION_RESOLVER(), context); if (o != null) { - ret.put(oField, o); + if(outputs.contains(oField)) { + ret.put(oField, o); + } + else { + intermediateVariables.put(oField, o); + } } } catch(Exception ex) { http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java index ba03b70..b4df514 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/FieldTransformationTest.java @@ -32,6 +32,7 @@ import org.junit.Assert; import org.junit.Test; import java.io.IOException; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -41,7 +42,7 @@ public class FieldTransformationTest { @Override public Map<String, Object> map( Map<String, Object> input , List<String> outputField - , Map<String, Object> fieldMappingConfig + , LinkedHashMap<String, Object> fieldMappingConfig , Map<String, Object> sensorConfig , Context context ) http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/1f83fdcb/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java index a517986..ac9b74c 100644 --- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java +++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java @@ -115,16 +115,48 @@ public class StellarTransformationTest { /** { - "fieldTransformations" : [ - { - "transformation" : "STELLAR" - ,"output" : ["newStellarField","utc_timestamp"] - ,"config" : { - "newStellarField" : "'<<??>>'", - "utc_timestamp" : "TO_EPOCH_TIMESTAMP(timestamp, 'yyyy-MM-dd HH:mm:ss', 'UTC')" - } + "fieldTransformations" : [ + { + "transformation" : "STELLAR" + ,"output" : "final_value" + ,"config" : { + "value1" : "1" + ,"value2" : "value1 + 1" + ,"final_value" : "value2 + 1" + } + } + ] } - ] + */ + @Multiline + public static String intermediateValuesConfig; + + @Test + public void testIntermediateValues() throws Exception { + + SensorParserConfig c = SensorParserConfig.fromBytes(Bytes.toBytes(intermediateValuesConfig)); + FieldTransformer handler = Iterables.getFirst(c.getFieldTransformations(), null); + JSONObject input = new JSONObject(new HashMap<String, Object>() {{ + }}); + handler.transformAndUpdate(input, new HashMap<>(), Context.EMPTY_CONTEXT()); + int expected = 3; + Assert.assertEquals(expected, input.get("final_value")); + Assert.assertFalse(input.containsKey("value1")); + Assert.assertFalse(input.containsKey("value2")); + } + + /** + { + "fieldTransformations" : [ + { + "transformation" : "STELLAR" + ,"output" : ["newStellarField","utc_timestamp"] + ,"config" : { + "newStellarField" : "'<<??>>'", + "utc_timestamp" : "TO_EPOCH_TIMESTAMP(timestamp, 'yyyy-MM-dd HH:mm:ss', 'UTC')" + } + } + ] } */ @Multiline @@ -163,6 +195,7 @@ public class StellarTransformationTest { Assert.assertEquals(expected, input.get("utc_timestamp")); Assert.assertTrue(input.containsKey("timestamp")); } + /** * Ensures that if we try to transform with a field which does not exist, it does not * 1. throw an exception
