Repository: incubator-metron Updated Branches: refs/heads/master c10a53eb8 -> 6834e146d
METRON-434: JSON Parser closes apache/incubator-metron#261 Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/6834e146 Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/6834e146 Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/6834e146 Branch: refs/heads/master Commit: 6834e146de163b5f6a441410a72fefb8db96944e Parents: c10a53e Author: cstella <ceste...@gmail.com> Authored: Sat Sep 24 01:09:43 2016 -0400 Committer: cstella <ceste...@gmail.com> Committed: Sat Sep 24 01:09:43 2016 -0400 ---------------------------------------------------------------------- .../data/jsonMap/parsed/jsonMapExampleParsed | 2 + .../data/jsonMap/raw/jsonMapExampleOutput | 2 + .../main/config/zookeeper/parsers/jsonMap.json | 4 + .../metron/parsers/json/JSONMapParser.java | 129 +++++++++++++++++++ .../integration/JSONMapIntegrationTest.java | 37 ++++++ .../metron/parsers/json/JSONMapParserTest.java | 112 ++++++++++++++++ 6 files changed, 286 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6834e146/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/parsed/jsonMapExampleParsed ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/parsed/jsonMapExampleParsed b/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/parsed/jsonMapExampleParsed new file mode 100644 index 0000000..795ed45 --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/parsed/jsonMapExampleParsed @@ -0,0 +1,2 @@ +{ "string" : "bar", "number" : 2, "ignored" : [ "blah" ], "original_string":"{ \"string\" : \"bar\", \"number\" : 2, \"ignored\" : [ \"blah\" ] }","timestamp":1000000000000, "source.type":"jsonMap" } +{ "number" : 7 , "original_string" : "{ \"number\" : 7 }", "source.type":"jsonMap","timestamp":1000000000000} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6834e146/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/raw/jsonMapExampleOutput ---------------------------------------------------------------------- diff --git a/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/raw/jsonMapExampleOutput b/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/raw/jsonMapExampleOutput new file mode 100644 index 0000000..8064084 --- /dev/null +++ b/metron-platform/metron-integration-test/src/main/sample/data/jsonMap/raw/jsonMapExampleOutput @@ -0,0 +1,2 @@ +{ "string" : "bar", "number" : 2, "ignored" : [ "blah" ] } +{ "number" : 7 } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6834e146/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json new file mode 100644 index 0000000..39e1da9 --- /dev/null +++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/jsonMap.json @@ -0,0 +1,4 @@ +{ + "parserClassName":"org.apache.metron.parsers.json.JSONMapParser", + "sensorTopic":"jsonMap" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6834e146/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java new file mode 100644 index 0000000..0bcf0f5 --- /dev/null +++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.json; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.parsers.BasicParser; +import org.json.simple.JSONObject; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class JSONMapParser extends BasicParser { + private static interface Handler { + JSONObject handle(String key, Map value, JSONObject obj); + } + public static enum MapStrategy implements Handler { + DROP((key, value, obj) -> obj) + ,UNFOLD( (key, value, obj) -> { + Set<Map.Entry<Object, Object>> entrySet = value.entrySet(); + for(Map.Entry<Object, Object> kv : entrySet) { + String newKey = Joiner.on(".").join(key, kv.getKey().toString()); + obj.put(newKey, kv.getValue()); + } + return obj; + }) + ,ALLOW((key, value, obj) -> { + obj.put(key, value); + return obj; + }) + ,ERROR((key, value, obj) -> { + throw new IllegalStateException("Unable to process " + key + " => " + value + " because value is a map."); + }) + ; + Handler handler; + MapStrategy(Handler handler) { + this.handler = handler; + } + + @Override + public JSONObject handle(String key, Map value, JSONObject obj) { + return handler.handle(key, value, obj); + } + + } + public static final String MAP_STRATEGY_CONFIG = "mapStrategy"; + private MapStrategy mapStrategy = MapStrategy.DROP; + + @Override + public void configure(Map<String, Object> config) { + String strategyStr = (String) config.getOrDefault(MAP_STRATEGY_CONFIG, MapStrategy.DROP.name()); + mapStrategy = MapStrategy.valueOf(strategyStr); + } + + /** + * Initialize the message parser. This is done once. + */ + @Override + public void init() { + + } + + /** + * Take raw data and convert it to a list of messages. + * + * @param rawMessage + * @return If null is returned, this is treated as an empty list. + */ + @Override + public List<JSONObject> parse(byte[] rawMessage) { + try { + String originalString = new String(rawMessage); + //convert the JSON blob into a String -> Object map + Map<String, Object> rawMap = JSONUtils.INSTANCE.load(originalString, new TypeReference<Map<String, Object>>() { + }); + JSONObject ret = normalizeJSON(rawMap); + ret.put("original_string", originalString ); + if(!ret.containsKey("timestamp")) { + //we have to ensure that we have a timestamp. This is one of the pre-requisites for the parser. + ret.put("timestamp", System.currentTimeMillis()); + } + return ImmutableList.of(ret); + } catch (Throwable e) { + String message = "Unable to parse " + new String(rawMessage) + ": " + e.getMessage(); + LOG.error(message, e); + throw new IllegalStateException(message, e); + } + } + + /** + * Process all sub-maps via the MapHandler. We have standardized on one-dimensional maps as our data model.. + * + * @param map + * @return + */ + private JSONObject normalizeJSON(Map<String, Object> map) { + JSONObject ret = new JSONObject(); + for(Map.Entry<String, Object> kv : map.entrySet()) { + if(kv.getValue() instanceof Map) { + mapStrategy.handle(kv.getKey(), (Map) kv.getValue(), ret); + } + else { + ret.put(kv.getKey(), kv.getValue()); + } + } + return ret; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6834e146/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapIntegrationTest.java new file mode 100644 index 0000000..13a7835 --- /dev/null +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/JSONMapIntegrationTest.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.integration; + +import org.apache.metron.parsers.integration.validation.SampleDataValidation; + +import java.util.ArrayList; +import java.util.List; + +public class JSONMapIntegrationTest extends ParserIntegrationTest { + @Override + String getSensorType() { + return "jsonMap"; + } + + @Override + List<ParserValidation> getValidations() { + return new ArrayList<ParserValidation>() {{ + add(new SampleDataValidation()); + }}; + } +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/6834e146/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java new file mode 100644 index 0000000..1299d97 --- /dev/null +++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/json/JSONMapParserTest.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.parsers.json; + +import com.google.common.collect.ImmutableMap; +import org.adrianwalker.multilinestring.Multiline; +import org.json.simple.JSONObject; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +public class JSONMapParserTest { + + /** + { + "foo" : "bar" + ,"blah" : "blah" + ,"number" : 2.0 + } + */ + @Multiline + static String happyPathJSON; + + @Test + public void testHappyPath() { + JSONMapParser parser = new JSONMapParser(); + List<JSONObject> output = parser.parse(happyPathJSON.getBytes()); + Assert.assertEquals(output.size(), 1); + //don't forget the timestamp field! + Assert.assertEquals(output.get(0).size(), 5); + JSONObject message = output.get(0); + Assert.assertEquals("bar", message.get("foo")); + Assert.assertEquals("blah", message.get("blah")); + Assert.assertNotNull(message.get("timestamp")); + Assert.assertTrue(message.get("timestamp") instanceof Number); + Assert.assertNotNull(message.get("number")); + Assert.assertTrue(message.get("number") instanceof Number); + } + + /** + { + "collection" : { "blah" : 7, "blah2" : "foo" } + } + */ + @Multiline + static String collectionHandlingJSON; + + @Test + public void testCollectionHandlingDrop() { + JSONMapParser parser = new JSONMapParser(); + List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes()); + Assert.assertEquals(output.size(), 1); + //don't forget the timestamp field! + Assert.assertEquals(output.get(0).size(), 2); + JSONObject message = output.get(0); + Assert.assertNotNull(message.get("timestamp")); + Assert.assertTrue(message.get("timestamp") instanceof Number); + } + + @Test(expected=IllegalStateException.class) + public void testCollectionHandlingError() { + JSONMapParser parser = new JSONMapParser(); + parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ERROR.name())); + parser.parse(collectionHandlingJSON.getBytes()); + } + + + @Test + public void testCollectionHandlingAllow() { + JSONMapParser parser = new JSONMapParser(); + parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.ALLOW.name())); + List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes()); + Assert.assertEquals(output.size(), 1); + //don't forget the timestamp field! + Assert.assertEquals(output.get(0).size(), 3); + JSONObject message = output.get(0); + Assert.assertNotNull(message.get("timestamp")); + Assert.assertTrue(message.get("timestamp") instanceof Number); + } + + @Test + public void testCollectionHandlingUnfold() { + JSONMapParser parser = new JSONMapParser(); + parser.configure(ImmutableMap.of(JSONMapParser.MAP_STRATEGY_CONFIG, JSONMapParser.MapStrategy.UNFOLD.name())); + List<JSONObject> output = parser.parse(collectionHandlingJSON.getBytes()); + Assert.assertEquals(output.size(), 1); + //don't forget the timestamp field! + Assert.assertEquals(output.get(0).size(), 4); + JSONObject message = output.get(0); + Assert.assertEquals(message.get("collection.blah"), 7); + Assert.assertEquals(message.get("collection.blah2"), "foo"); + Assert.assertNotNull(message.get("timestamp")); + Assert.assertTrue(message.get("timestamp") instanceof Number); + } +}