This is an automated email from the ASF dual-hosted git repository.
mmiklavcic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new dc273a5 METRON-2071 Add MAP_PUT and MAP_MERGE to Stellar (mmiklavc)
closes apache/metron#1385
dc273a5 is described below
commit dc273a5be628c458ad0b25380d55536cd37b7a4f
Author: mmiklavc <michael.miklav...@gmail.com>
AuthorDate: Tue Apr 16 12:37:16 2019 -0600
METRON-2071 Add MAP_PUT and MAP_MERGE to Stellar (mmiklavc) closes
apache/metron#1385
---
metron-stellar/stellar-common/README.md | 26 ++++--
.../metron/stellar/dsl/functions/MapFunctions.java | 93 +++++++++++++++++---
.../stellar/dsl/functions/BasicStellarTest.java | 99 ++++++++++++++++++++++
3 files changed, 201 insertions(+), 17 deletions(-)
diff --git a/metron-stellar/stellar-common/README.md
b/metron-stellar/stellar-common/README.md
index 8301f6d..261be59 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -165,7 +165,7 @@ Where:
| ----------
|
| [ `ABS`](../../metron-analytics/metron-statistics#abs)
|
| [ `APPEND_IF_MISSING`](#append_if_missing)
|
-| [ `ASN_GET`](#asn_get)
|
+| [ `ASN_GET`](#asn_get)
|
| [ `BIN`](../../metron-analytics/metron-statistics#bin)
|
| [ `BLOOM_ADD`](#bloom_add)
|
| [ `BLOOM_EXISTS`](#bloom_exists)
|
@@ -176,7 +176,7 @@ Where:
| [ `CHOP`](#chop)
|
| [ `CHOMP`](#chomp)
|
| [ `COUNT_MATCHES`](#count_matches)
|
-| [ `DATE_FORMAT`](#date_format)
+| [ `DATE_FORMAT`](#date_format)
|
| [ `DAY_OF_MONTH`](#day_of_month)
|
| [ `DAY_OF_WEEK`](#day_of_week)
|
| [ `DAY_OF_YEAR`](#day_of_year)
|
@@ -238,6 +238,8 @@ Where:
| [ `MAP`](#map)
|
| [ `MAP_EXISTS`](#map_exists)
|
| [ `MAP_GET`](#map_get)
|
+| [ `MAP_MERGE`](#map_merge)
|
+| [ `MAP_PUT`](#map_put)
|
| [ `MAX`](#MAX)
|
| [ `MIN`](#MIN)
|
| [ `MONTH`](#month)
|
@@ -249,15 +251,15 @@ Where:
| [ `OBJECT_GET`](#object_get)
|
| [ `PREPEND_IF_MISSING`](#prepend_if_missing)
|
| [ `PROFILE_GET`](#profile_get)
|
-| [ `PROFILE_VERBOSE`](#profile_verbose)
|
+| [ `PROFILE_VERBOSE`](#profile_verbose)
|
| [ `PROFILE_FIXED`](#profile_fixed)
|
| [ `PROFILE_WINDOW`](#profile_window)
|
| [ `PROTOCOL_TO_NAME`](#protocol_to_name)
|
| [ `REDUCE`](#reduce)
|
| [ `REGEXP_MATCH`](#regexp_match)
|
| [ `REGEXP_GROUP_VAL`](#regexp_group_val)
|
-| [ `REGEXP_REPLACE`](#regexp_replace)
-| [ `REST_GET`](#rest_get)
+| [ `REGEXP_REPLACE`](#regexp_replace)
|
+| [ `REST_GET`](#rest_get)
|
| [ `ROUND`](#round)
|
| [ `SAMPLE_ADD`](../../metron-analytics/metron-statistics#sample_add)
|
| [ `SAMPLE_GET`](../../metron-analytics/metron-statistics#sample_get)
|
@@ -804,6 +806,20 @@ Where:
* default - Optionally the default value to return if the key is not in
the map.
* Returns: The object associated with the key in the map. If no value is
associated with the key and default is specified, then default is returned. If
no value is associated with the key or default, then null is returned.
+### `MAP_MERGE`
+ * Description: Merges a list of maps
+ * Input:
+ * maps - A collection of maps to merge. Last entry wins for overlapping
keys.
+ * Returns: A Map. null if the list of maps is empty.
+
+### `MAP_PUT`
+ * Description: Adds a key/value pair to a map
+ * Input:
+ * key - The key
+ * value - The value
+ * map - The map to perform the put on
+ * Returns: The original map modified with the key/value. If the map argument
is null, a new map will be created and returned that contains the provided key
and value - note: if the 'map' argument is null, only the returned map will be
non-null and contain the key/value.
+
### `MAX`
* Description: Returns the maximum value of a list of input values.
* Input:
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/MapFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/MapFunctions.java
index 34d442e..5bf109a 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/MapFunctions.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/MapFunctions.java
@@ -18,11 +18,13 @@
package org.apache.metron.stellar.dsl.functions;
-import org.apache.metron.stellar.dsl.BaseStellarFunction;
-import org.apache.metron.stellar.dsl.Stellar;
-
+import com.google.common.collect.Iterables;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import org.apache.metron.stellar.dsl.BaseStellarFunction;
+import org.apache.metron.stellar.dsl.Stellar;
public class MapFunctions {
@@ -52,15 +54,15 @@ public class MapFunctions {
}
@Stellar(name="GET"
- ,namespace="MAP"
- , description="Gets the value associated with a key from a map"
- , params = {
- "key - The key"
- ,"map - The map"
- ,"default - Optionally the default value to return if the
key is not in the map."
- }
- , returns = "The object associated with the key in the map. If no
value is associated with the key and default is specified, then default is
returned. If no value is associated with the key or default, then null is
returned."
- )
+ ,namespace="MAP"
+ , description="Gets the value associated with a key from a map"
+ , params = {
+ "key - The key"
+ ,"map - The map"
+ ,"default - Optionally the default value to return if the key is not in
the map."
+ }
+ , returns = "The object associated with the key in the map. If no value
is associated with the key and default is specified, then default is returned.
If no value is associated with the key or default, then null is returned."
+ )
public static class MapGet extends BaseStellarFunction {
@Override
@SuppressWarnings("unchecked")
@@ -82,4 +84,71 @@ public class MapFunctions {
return ret;
}
}
+
+ @Stellar(name = "PUT",
+ namespace = "MAP",
+ description = "Adds a key/value pair to a map",
+ params = {
+ "key - The key",
+ "value - The value",
+ "map - The map to perform the put on"
+ },
+ returns = "The original map modified with the key/value. If the map
argument is null, a new map will be created and returned that contains the
provided key and value - note: if the 'map' argument is null, only the returned
map will be non-null and contain the key/value."
+ )
+ public static class MapPut extends BaseStellarFunction {
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Object apply(List<Object> objects) {
+ if (objects.size() < 3) {
+ throw new IllegalArgumentException("Must pass a key, value, and map");
+ } else {
+ Object keyObj = objects.get(0);
+ Object valueObj = objects.get(1);
+ Object mapObj = objects.get(2);
+ if (mapObj == null) {
+ mapObj = new HashMap<>();
+ }
+ Map<Object, Object> map = (Map) mapObj;
+ map.put(keyObj, valueObj);
+ return map;
+ }
+ }
+ }
+
+ @Stellar(name = "MERGE",
+ namespace = "MAP",
+ description = "Merges a list of maps",
+ params = {"maps - A collection of maps to merge. Last entry wins
for overlapping keys."},
+ returns = "A Map. null if the list of maps is empty."
+ )
+ public static class MapMerge extends BaseStellarFunction {
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Object apply(List<Object> list) {
+ if (list.size() < 1) {
+ return null;
+ }
+ LinkedHashMap<Object, Object> ret = new LinkedHashMap<>();
+ Object o = list.get(0);
+ if (o != null) {
+ if (!(o instanceof Iterable)) {
+ throw new IllegalArgumentException("Expected an Iterable, but " + o
+ " is of type " + o.getClass());
+ }
+ Iterable<? extends Map> maps = (Iterable<? extends Map>) o;
+
+ if (Iterables.size(maps) == 1) {
+ return Iterables.getFirst(maps, null);
+ }
+
+ for (Map m : maps) {
+ if (m != null) {
+ ret.putAll(m);
+ }
+ }
+ }
+ return ret;
+ }
+ }
}
diff --git
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
index a6a41e5..4b64f72 100644
---
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
+++
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/BasicStellarTest.java
@@ -21,6 +21,9 @@ package org.apache.metron.stellar.dsl.functions;
import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
import static
org.apache.metron.stellar.common.utils.StellarProcessorUtils.runPredicate;
import static
org.apache.metron.stellar.common.utils.StellarProcessorUtils.validate;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.junit.Assert.assertThat;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
@@ -35,6 +38,7 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.metron.stellar.common.StellarProcessor;
+import org.apache.metron.stellar.common.utils.StellarProcessorUtils;
import org.apache.metron.stellar.dsl.Context;
import org.apache.metron.stellar.dsl.DefaultVariableResolver;
import org.apache.metron.stellar.dsl.MapVariableResolver;
@@ -642,6 +646,101 @@ public class BasicStellarTest {
}
@Test
+ public void testMapPut() {
+ Map vars = ImmutableMap.of("mymap", new HashMap<String, String>());
+ String query = "MAP_PUT('foo','bar',mymap)";
+ assertThat(run(query, vars), instanceOf(Map.class));
+ query = "MAP_GET('foo', mymap)";
+ assertThat(run(query, vars), equalTo("bar"));
+ }
+
+ @Test
+ public void testMapPutDefault() {
+ Map vars = new HashMap() {{
+ put("mymap", null);
+ }};
+ String query = "MAP_PUT('foo','bar', mymap)";
+ Map result = (Map) run(query, vars);
+ assertThat(result, instanceOf(Map.class));
+ assertThat(result.size(), equalTo(1));
+ assertThat(result.get("foo"), equalTo("bar"));
+ }
+
+ @Test(expected=ParseException.class)
+ public void mapPutTest_wrongType() throws Exception {
+ Map s = (Map) run("MAP_PUT( 'foo', 'bar', [ 'baz' ] )", new HashMap<>());
+ }
+
+ @Test
+ public void testMapMergeEmpty() {
+ Map m = (Map) StellarProcessorUtils.run("MAP_MERGE([{}, null])", new
HashMap<>());
+ Assert.assertEquals(0, m.size());
+ }
+
+ @Test
+ public void testMapMergeFromVariables() {
+ Map vars = new HashMap() {{
+ put("map1", ImmutableMap.of("a", 1, "b", 2));
+ put("map2", ImmutableMap.of("c", 3, "d", 4));
+ put("map3", ImmutableMap.of("e", 5, "f", 6));
+ }};
+ String query = "MAP_MERGE([map1, map2, map3])";
+ Map result = (Map) run(query, vars);
+ assertThat(result, instanceOf(Map.class));
+ assertThat(result.size(), equalTo(6));
+ assertThat(result.get("a"), equalTo(1));
+ assertThat(result.get("b"), equalTo(2));
+ assertThat(result.get("c"), equalTo(3));
+ assertThat(result.get("d"), equalTo(4));
+ assertThat(result.get("e"), equalTo(5));
+ assertThat(result.get("f"), equalTo(6));
+ }
+
+ @Test
+ public void testMapMergeSingleMap() {
+ String query = "MAP_MERGE( [ { 'a' : '1', 'b' : '2', 'c' : '3' } ] )";
+ Map result = (Map) run(query, new HashMap<>());
+ assertThat(result, instanceOf(Map.class));
+ assertThat(result.size(), equalTo(3));
+ assertThat(result.get("a"), equalTo("1"));
+ assertThat(result.get("b"), equalTo("2"));
+ assertThat(result.get("c"), equalTo("3"));
+ }
+
+ @Test
+ public void testMapMergeFromInlineMaps() {
+ String query = "MAP_MERGE( [ { 'a' : '1', 'b' : '2' }, { 'c' : '3', 'd' :
'4' }, { 'e' : '5', 'f' : '6' } ] )";
+ Map result = (Map) run(query, new HashMap<>());
+ assertThat(result, instanceOf(Map.class));
+ assertThat(result.size(), equalTo(6));
+ assertThat(result.get("a"), equalTo("1"));
+ assertThat(result.get("b"), equalTo("2"));
+ assertThat(result.get("c"), equalTo("3"));
+ assertThat(result.get("d"), equalTo("4"));
+ assertThat(result.get("e"), equalTo("5"));
+ assertThat(result.get("f"), equalTo("6"));
+ }
+
+ @Test
+ public void testMapMergeWithOverlappingMapsAndMixedTypes() {
+ String query = "MAP_MERGE( [ { 'a' : '1', 'b' : 2, 'c' : '3' }, { 'c' :
'3b', 'd' : '4' }, { 'd' : '4b', 'e' : 5, 'f' : '6' } ] )";
+ Map result = (Map) run(query, new HashMap<>());
+ assertThat(result, instanceOf(Map.class));
+ assertThat(result.size(), equalTo(6));
+ assertThat(result.get("a"), equalTo("1"));
+ assertThat(result.get("b"), equalTo(2));
+ assertThat(result.get("c"), equalTo("3b"));
+ assertThat(result.get("d"), equalTo("4b"));
+ assertThat(result.get("e"), equalTo(5));
+ assertThat(result.get("f"), equalTo("6"));
+ }
+
+ @Test(expected=ParseException.class)
+ public void mapMergeTest_wrongType() throws Exception {
+ Map s = (Map) run("MAP_MERGE( [ 'foo', 'bar' ] )", new HashMap<>());
+ }
+
+ @Test
public void testTLDExtraction() {
String query = "DOMAIN_TO_TLD(foo)";
Assert.assertEquals("co.uk", run(query, ImmutableMap.of("foo",
"www.google.co.uk")));
