Repository: metron Updated Branches: refs/heads/master 222685760 -> d64afbc3f
METRON-1115 Jackson is not properly serializing readMetadata and mergeMetadata SensorParserConfig properties (merrimanr) closes apache/metron#703 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d64afbc3 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d64afbc3 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d64afbc3 Branch: refs/heads/master Commit: d64afbc3f38e81d462d45f99efb21deec49e6b55 Parents: 2226857 Author: merrimanr <[email protected]> Authored: Mon Aug 21 15:05:47 2017 -0500 Committer: merrimanr <[email protected]> Committed: Mon Aug 21 15:05:47 2017 -0500 ---------------------------------------------------------------------- .../SensorParserConfigControllerIntegrationTest.java | 13 ++++++++++++- .../common/configuration/SensorParserConfig.java | 12 ++++++------ .../metron/management/ConfigurationFunctionsTest.java | 4 +++- .../org/apache/metron/parsers/bolt/ParserBolt.java | 6 +++--- 4 files changed, 24 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/d64afbc3/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java index 45ee1f5..f5ea23d 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java @@ -39,6 +39,7 @@ import java.io.File; import java.io.IOException; import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.hamcrest.Matchers.hasSize; import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; @@ -82,7 +83,9 @@ public class SensorParserConfigControllerIntegrationTest { { "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser", "sensorTopic":"broTest", - "parserConfig": {} + "parserConfig": {}, + "readMetadata": true, + "mergeMetadata": true } */ @Multiline @@ -197,6 +200,7 @@ public class SensorParserConfigControllerIntegrationTest { this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(squidJson)) .andExpect(status().isCreated()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.*", hasSize(10))) .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.GrokParser")) .andExpect(jsonPath("$.sensorTopic").value("squidTest")) .andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest")) @@ -211,6 +215,7 @@ public class SensorParserConfigControllerIntegrationTest { this.mockMvc.perform(get(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password))) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.*", hasSize(10))) .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.GrokParser")) .andExpect(jsonPath("$.sensorTopic").value("squidTest")) .andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest")) @@ -239,15 +244,21 @@ public class SensorParserConfigControllerIntegrationTest { this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) .andExpect(status().isCreated()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.*", hasSize(10))) .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.bro.BasicBroParser")) .andExpect(jsonPath("$.sensorTopic").value("broTest")) + .andExpect(jsonPath("$.readMetadata").value("true")) + .andExpect(jsonPath("$.mergeMetadata").value("true")) .andExpect(jsonPath("$.parserConfig").isEmpty()); this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.*", hasSize(10))) .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.bro.BasicBroParser")) .andExpect(jsonPath("$.sensorTopic").value("broTest")) + .andExpect(jsonPath("$.readMetadata").value("true")) + .andExpect(jsonPath("$.mergeMetadata").value("true")) .andExpect(jsonPath("$.parserConfig").isEmpty()); this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password))) http://git-wip-us.apache.org/repos/asf/metron/blob/d64afbc3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java index d72f462..f08e9c4 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorParserConfig.java @@ -38,7 +38,7 @@ public class SensorParserConfig implements Serializable { private Boolean readMetadata = false; private Boolean mergeMetadata = false; - public Boolean mergeMetadata() { + public Boolean getMergeMetadata() { return mergeMetadata; } @@ -46,7 +46,7 @@ public class SensorParserConfig implements Serializable { this.mergeMetadata = mergeMetadata; } - public Boolean readMetadata() { + public Boolean getReadMetadata() { return readMetadata; } @@ -173,9 +173,9 @@ public class SensorParserConfig implements Serializable { return false; if (getParserConfig() != null ? !getParserConfig().equals(that.getParserConfig()) : that.getParserConfig() != null) return false; - if (readMetadata() != null ? !readMetadata().equals(that.readMetadata()) : that.readMetadata() != null) + if (getReadMetadata() != null ? !getReadMetadata().equals(that.getReadMetadata()) : that.getReadMetadata() != null) return false; - if (mergeMetadata() != null ? !mergeMetadata().equals(that.mergeMetadata()) : that.mergeMetadata() != null) + if (getMergeMetadata() != null ? !getMergeMetadata().equals(that.getMergeMetadata()) : that.getMergeMetadata() != null) return false; return getFieldTransformations() != null ? getFieldTransformations().equals(that.getFieldTransformations()) : that.getFieldTransformations() == null; @@ -191,8 +191,8 @@ public class SensorParserConfig implements Serializable { result = 31 * result + (getInvalidWriterClassName() != null ? getInvalidWriterClassName().hashCode() : 0); result = 31 * result + (getParserConfig() != null ? getParserConfig().hashCode() : 0); result = 31 * result + (getFieldTransformations() != null ? getFieldTransformations().hashCode() : 0); - result = 31 * result + (readMetadata() != null ? readMetadata().hashCode() : 0); - result = 31 * result + (mergeMetadata() != null ? mergeMetadata().hashCode() : 0); + result = 31 * result + (getReadMetadata() != null ? getReadMetadata().hashCode() : 0); + result = 31 * result + (getMergeMetadata() != null ? getMergeMetadata().hashCode() : 0); return result; } } http://git-wip-us.apache.org/repos/asf/metron/blob/d64afbc3/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java index ade5943..431ece2 100644 --- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java +++ b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ConfigurationFunctionsTest.java @@ -79,7 +79,9 @@ public class ConfigurationFunctionsTest { { "sensorTopic" : "brop", "parserConfig" : { }, - "fieldTransformations" : [ ] + "fieldTransformations" : [ ], + "readMetadata":false, + "mergeMetadata":false } */ @Multiline http://git-wip-us.apache.org/repos/asf/metron/blob/d64afbc3/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java index b3b9586..99785b2 100644 --- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java @@ -172,17 +172,17 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable { boolean ackTuple = !writer.handleAck(); int numWritten = 0; if(sensorParserConfig != null) { - Map<String, Object> metadata = getMetadata(tuple, sensorParserConfig.readMetadata()); + Map<String, Object> metadata = getMetadata(tuple, sensorParserConfig.getReadMetadata()); List<FieldValidator> fieldValidations = getConfigurations().getFieldValidations(); Optional<List<JSONObject>> messages = parser.parseOptional(originalMessage); for (JSONObject message : messages.orElse(Collections.emptyList())) { message.put(Constants.SENSOR_TYPE, getSensorType()); - if(sensorParserConfig.mergeMetadata()) { + if(sensorParserConfig.getMergeMetadata()) { message.putAll(metadata); } for (FieldTransformer handler : sensorParserConfig.getFieldTransformations()) { if (handler != null) { - if(!sensorParserConfig.mergeMetadata()) { + if(!sensorParserConfig.getMergeMetadata()) { //if we haven't merged metadata, then we need to pass them along as configuration params. handler.transformAndUpdate(message, stellarContext, sensorParserConfig.getParserConfig(), metadata); }
