Repository: metron Updated Branches: refs/heads/master 95db8b40e -> 309d3757d
METRON-1146: Add ability to parse JSON string into JSONObject for stellar closes apache/incubator-metron#727 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/309d3757 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/309d3757 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/309d3757 Branch: refs/heads/master Commit: 309d3757df50a2dda0ac4396750e292e76926c5d Parents: 95db8b4 Author: Anand Subramanian <[email protected]> Authored: Tue Sep 12 09:27:09 2017 -0700 Committer: cstella <[email protected]> Committed: Tue Sep 12 09:27:09 2017 -0700 ---------------------------------------------------------------------- metron-stellar/stellar-common/README.md | 26 +- .../stellar/dsl/functions/StringFunctions.java | 115 ++++++++ .../dsl/functions/StringFunctionsTest.java | 261 +++++++++++++++++++ 3 files changed, 401 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/309d3757/metron-stellar/stellar-common/README.md ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md index 8552138..d464d8c 100644 --- a/metron-stellar/stellar-common/README.md +++ b/metron-stellar/stellar-common/README.md @@ -224,6 +224,9 @@ In the core language functions, we support basic functional programming primitiv | [ `TO_EPOCH_TIMESTAMP`](#to_epoch_timestamp) | | [ `TO_FLOAT`](#to_float) | | [ `TO_INTEGER`](#to_integer) | +| [ `TO_JSON_LIST`](#to_json_List) | +| [ `TO_JSON_MAP`](#to_json_map) | +| [ `TO_JSON_OBJECT`](#to_json_object) | | [ `TO_LONG`](#to_long) | | [ `TO_LOWER`](#to_lower) | | [ `TO_STRING`](#to_string) | @@ -638,6 +641,27 @@ In the core language functions, we support basic functional programming primitiv * Description: Returns a list of the encodings that are currently supported. * Returns: A List of String +### `TO_JSON_LIST` + * Description: Accepts JSON string as an input and returns a List object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed. + For e.g. `GET_FIRST( TO_JSON_LIST( '[ "foo", 2]')` would yield `foo` + * Input: + * string - The JSON string to be parsed + * Returns: A parsed List object + +### `TO_JSON_MAP` + * Description: Accepts JSON string as an input and returns a Map object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed. + For e.g. `MAP_GET( 'bar', TO_JSON_MAP( '{ "foo" : 1, "bar" : 2}' )` would yield `2` + * Input: + * string - The JSON string to be parsed + * Returns: A parsed Map object + +### `TO_JSON_OBJECT` + * Description: Accepts JSON string as an input and returns a JSON Object parsed by Jackson. You need to be aware of content of JSON string that is to be parsed. + For e.g. `MAP_GET( 'bar', TO_JSON_OBJECT( '{ "foo" : 1, "bar" : 2}' )` would yield `2` + * Input: + * string - The JSON string to be parsed + * Returns: A parsed JSON object + ### `LOG2` * Description: Returns the log (base `2`) of a number. * Input: @@ -698,7 +722,7 @@ In the core language functions, we support basic functional programming primitiv * Input: * dateTime - The datetime as a long representing the milliseconds since unix epoch * Returns: The current month (0-based). - + ### `MULTISET_ADD` * Description: Adds to a multiset, which is a map associating objects to their instance counts. * Input: http://git-wip-us.apache.org/repos/asf/metron/blob/309d3757/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java index 7f4f396..4dc4790 100644 --- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java +++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java @@ -18,15 +18,20 @@ package org.apache.metron.stellar.dsl.functions; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import org.apache.commons.lang3.StringUtils; +import org.apache.metron.stellar.common.utils.JSONUtils; import org.apache.metron.stellar.dsl.BaseStellarFunction; import org.apache.metron.stellar.dsl.ParseException; import org.apache.metron.stellar.dsl.Stellar; import org.apache.metron.stellar.common.utils.ConversionUtils; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -502,4 +507,114 @@ public class StringFunctions { } } + @Stellar(name = "TO_JSON_OBJECT" + , description = "Returns a JSON object for the specified JSON string" + , params = { + "str - the JSON String to convert, may be null" + } + , returns = "an Object containing the parsed JSON string" + ) + public static class ToJsonObject extends BaseStellarFunction { + + @Override + public Object apply(List<Object> strings) { + + if (strings == null || strings.size() == 0) { + throw new IllegalArgumentException("[TO_JSON_OBJECT] incorrect arguments. Usage: TO_JSON_OBJECT <String>"); + } + String var = (strings.get(0) == null) ? null : (String) strings.get(0); + if (var == null) { + return null; + } else if (var.length() == 0) { + return var; + } else { + if (!(strings.get(0) instanceof String)) { + throw new ParseException("Valid JSON string not supplied"); + } + // Return JSON Object + try { + return JSONUtils.INSTANCE.load((String) strings.get(0), Object.class); + } catch (JsonProcessingException ex) { + throw new ParseException("Valid JSON string not supplied", ex); + } catch (IOException e) { + e.printStackTrace(); + } + } + return new ParseException("Unable to parse JSON string"); + } + } + + @Stellar(name = "TO_JSON_MAP" + , description = "Returns a MAP object for the specified JSON string" + , params = { + "str - the JSON String to convert, may be null" + } + , returns = "a MAP object containing the parsed JSON string" + ) + public static class ToJsonMap extends BaseStellarFunction { + + @Override + public Object apply(List<Object> strings) { + + if (strings == null || strings.size() == 0) { + throw new IllegalArgumentException("[TO_JSON_MAP] incorrect arguments. Usage: TO_JSON_MAP <JSON String>"); + } + String var = (strings.get(0) == null) ? null : (String) strings.get(0); + if (var == null) { + return null; + } else if (var.length() == 0) { + return var; + } else { + if (!(strings.get(0) instanceof String)) { + throw new ParseException("Valid JSON string not supplied"); + } + // Return parsed JSON Object as a HashMap + try { + return JSONUtils.INSTANCE.load((String) strings.get(0), new TypeReference<Map<String, Object>>(){}); + } catch (JsonProcessingException ex) { + throw new ParseException("Valid JSON string not supplied", ex); + } catch (IOException e) { + e.printStackTrace(); + } + } + return new ParseException("Unable to parse JSON string"); + } + } + + @Stellar(name = "TO_JSON_LIST" + , description = "Returns a List object for the specified JSON string" + , params = { + "str - the JSON String to convert, may be null" + } + , returns = "a List object containing the parsed JSON string" + ) + public static class ToJsonList extends BaseStellarFunction { + + @Override + public Object apply(List<Object> strings) { + + if (strings == null || strings.size() == 0) { + throw new IllegalArgumentException("[TO_JSON_LIST] incorrect arguments. Usage: TO_JSON_LIST <JSON String>"); + } + String var = (strings.get(0) == null) ? null : (String) strings.get(0); + if (var == null) { + return null; + } else if (var.length() == 0) { + return var; + } else { + if (!(strings.get(0) instanceof String)) { + throw new ParseException("Valid JSON string not supplied"); + } + // Return parsed JSON Object as a List + try { + return (List) JSONUtils.INSTANCE.load((String) strings.get(0), new TypeReference<List<Object>>(){}); + } catch (JsonProcessingException ex) { + throw new ParseException("Valid JSON string not supplied", ex); + } catch (IOException e) { + e.printStackTrace(); + throw new ParseException("Valid JSON string not supplied", e); + } + } + } + } } http://git-wip-us.apache.org/repos/asf/metron/blob/309d3757/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java index cb33eaa..418bf2d 100644 --- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java +++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java @@ -20,15 +20,18 @@ package org.apache.metron.stellar.dsl.functions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.adrianwalker.multilinestring.Multiline; import org.apache.commons.collections4.map.HashedMap; import org.apache.metron.stellar.dsl.DefaultVariableResolver; import org.apache.metron.stellar.dsl.ParseException; import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run; @@ -484,4 +487,262 @@ public class StringFunctionsTest { Assert.assertTrue(thrown); } + + /** + * TO_JSON_OBJECT StringFunction + */ + + // Input strings to be used + /** + { "foo" : 2 } + */ + @Multiline + private String string1; + + /** + { + "foo" : "abc", + "bar" : "def" + } + */ + @Multiline + private String string2; + + /** + [ "foo", 2 ] + */ + @Multiline + private String string3; + + /** + [ "foo", "bar", "car" ] + */ + @Multiline + private String string4; + + /** + [ + { + "foo1":"abc", + "bar1":"def" + }, + { + "foo2":"ghi", + "bar2":"jkl" + } + ] + */ + @Multiline + private String string5; + + @Test + public void testToJsonObject() throws Exception { + //JSON Object + Object ret1 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string1)); + Assert.assertNotNull(ret1); + Assert.assertTrue (ret1 instanceof HashMap); + + Object ret2 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string2)); + Assert.assertNotNull(ret2); + Assert.assertTrue (ret2 instanceof HashMap); + Assert.assertEquals("def", run("MAP_GET( 'bar', returnval)", ImmutableMap.of("returnval", ret2))); + + //Simple Arrays + Object ret3 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string3)); + Assert.assertNotNull(ret3); + Assert.assertTrue (ret3 instanceof ArrayList); + List<Object> result3 = (List<Object>) ret3; + Assert.assertEquals(2, result3.get(1)); + + Object ret4 = run("TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string4)); + Assert.assertNotNull(ret4); + Assert.assertTrue (ret4 instanceof ArrayList); + List<Object> result4 = (List<Object>) ret4; + Assert.assertEquals("car", result4.get(2)); + + //JSON Array + Object ret5 = run( "TO_JSON_OBJECT(msg)", ImmutableMap.of("msg", string5)); + Assert.assertNotNull(ret5); + Assert.assertTrue (ret5 instanceof ArrayList); + List<List<Object>> result5 = (List<List<Object>>) ret5; + HashMap<String,String> results5Map1 = (HashMap) result5.get(0); + Assert.assertEquals("def", results5Map1.get("bar1")); + HashMap<String,String> results5Map2 = (HashMap) result5.get(1); + Assert.assertEquals("ghi", results5Map2.get("foo2")); + + // No input + boolean thrown = false; + try { + run("TO_JSON_OBJECT()", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Unable to parse")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Invalid input + try { + run("TO_JSON_OBJECT('123, 456')", new HashedMap<>()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Malformed JSON String + try { + run("TO_JSON_OBJECT('{\"foo\" : 2')", new HashedMap<>()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied")); + } + Assert.assertTrue(thrown); + thrown = false; + } + + @Test + public void testToJsonMap() throws Exception { + //JSON Object + Object ret1 = run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string1)); + Assert.assertNotNull(ret1); + Assert.assertTrue (ret1 instanceof HashMap); + + Object ret2 = run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string2)); + Assert.assertNotNull(ret2); + Assert.assertTrue (ret2 instanceof HashMap); + Assert.assertEquals("def", run("MAP_GET( 'bar', returnval)", ImmutableMap.of("returnval", ret2))); + + //Simple Arrays + boolean thrown = false; + try { + run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string3)); + } catch (ParseException pe) { + thrown = true; + } + Assert.assertTrue(thrown); + + thrown = false; + try { + run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string4)); + } catch (ParseException pe) { + thrown = true; + } + Assert.assertTrue (thrown); + + //JSON Array + thrown = false; + try { + run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string5)); + } catch (ParseException pe) { + thrown = true; + } + Assert.assertTrue(thrown); + + + // No input + try { + run("TO_JSON_MAP()", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Unable to parse")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Invalid input + try { + run("TO_JSON_MAP('123, 456')", new HashedMap<>()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied")); + } + Assert.assertTrue(thrown); + thrown = false; + + // Malformed JSON String + try { + run("TO_JSON_MAP('{\"foo\" : 2')", new HashedMap<>()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied")); + } + Assert.assertTrue(thrown); + thrown = false; + } + + @Test + public void testToJsonList() throws Exception { + //Simple Arrays + Object ret3 = run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string3)); + Assert.assertNotNull(ret3); + Assert.assertTrue (ret3 instanceof ArrayList); + List<Object> result3 = (List<Object>) ret3; + Assert.assertEquals(2, result3.get(1)); + + Object ret4 = run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string4)); + Assert.assertNotNull(ret4); + Assert.assertTrue (ret4 instanceof ArrayList); + List<Object> result4 = (List<Object>) ret4; + Assert.assertEquals("car", result4.get(2)); + + //JSON Array + Object ret5 = run( "TO_JSON_LIST(msg)", ImmutableMap.of("msg", string5)); + Assert.assertNotNull(ret5); + Assert.assertTrue (ret5 instanceof ArrayList); + List<List<Object>> result5 = (List<List<Object>>) ret5; + HashMap<String,String> results5Map1 = (HashMap) result5.get(0); + Assert.assertEquals("def", results5Map1.get("bar1")); + HashMap<String,String> results5Map2 = (HashMap) result5.get(1); + Assert.assertEquals("ghi", results5Map2.get("foo2")); + + //JSON Object - throws exception + boolean thrown = false; + try { + run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string1)); + } catch (ParseException pe) { + thrown = true; + } + Assert.assertTrue(thrown); + + thrown = false; + try { + run("TO_JSON_LIST(msg)", ImmutableMap.of("msg", string2)); + } catch (ParseException pe) { + thrown = true; + } + Assert.assertTrue (thrown); + + // No input + thrown = false; + try { + run("TO_JSON_LIST()", Collections.emptyMap()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Unable to parse")); + } + Assert.assertTrue(thrown); + + // Invalid input + thrown = false; + try { + run("TO_JSON_LIST('123, 456')", new HashedMap<>()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied")); + } + Assert.assertTrue(thrown); + + // Malformed JSON String + thrown = false; + try { + run("TO_JSON_LIST('{\"foo\" : 2')", new HashedMap<>()); + } catch (ParseException pe) { + thrown = true; + Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied")); + } + Assert.assertTrue(thrown); + } + }
