Repository: incubator-metron Updated Branches: refs/heads/master 9a5e1ba73 -> 345eba8d5
METRON-932: Change HLLP Stellar functions to accept empty lists (mmiklavc) closes apache/incubator-metron#566 Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/345eba8d Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/345eba8d Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/345eba8d Branch: refs/heads/master Commit: 345eba8d5940f2789e5a860eb51e8fa9f003c676 Parents: 9a5e1ba Author: mmiklavc <[email protected]> Authored: Fri May 12 11:59:28 2017 -0600 Committer: Michael Miklavcic <[email protected]> Committed: Fri May 12 11:59:28 2017 -0600 ---------------------------------------------------------------------- metron-analytics/metron-statistics/README.md | 4 ++-- .../approximation/HyperLogLogPlusFunctions.java | 20 +++++++++++++------- ...HyperLogLogPlusFunctionsIntegrationTest.java | 13 +++++++++++++ .../HyperLogLogPlusFunctionsTest.java | 17 +++++++++++++---- 4 files changed, 41 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/345eba8d/metron-analytics/metron-statistics/README.md ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/README.md b/metron-analytics/metron-statistics/README.md index cfd44f2..74e78b0 100644 --- a/metron-analytics/metron-statistics/README.md +++ b/metron-analytics/metron-statistics/README.md @@ -23,7 +23,7 @@ functions can be used from everywhere where Stellar is used. * Description: Returns HyperLogLogPlus-estimated cardinality for this set. See [HLLP README](HLLP.md) * Input: * hyperLogLogPlus - the hllp set - * Returns: Long value representing the cardinality for this set + * Returns: Long value representing the cardinality for this set. Cardinality of a null set is 0. #### `HLLP_INIT` * Description: Initializes the HyperLogLogPlus estimator set. p must be a value between 4 and sp and sp must be less than 32 and greater than 4. See [HLLP README](HLLP.md) @@ -36,7 +36,7 @@ functions can be used from everywhere where Stellar is used. * Description: Merge hllp sets together. The resulting estimator is initialized with p and sp precision values from the first provided hllp estimator set. See [HLLP README](HLLP.md) * Input: * hllp - List of hllp estimators to merge. Takes a single hllp set or a list. - * Returns: A new merged HyperLogLogPlus estimator set + * Returns: A new merged HyperLogLogPlus estimator set. Passing an empty list returns null. ### Mathematical Functions http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/345eba8d/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctions.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctions.java b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctions.java index 7578403..f123494 100644 --- a/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctions.java +++ b/metron-analytics/metron-statistics/src/main/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctions.java @@ -61,16 +61,22 @@ public class HyperLogLogPlusFunctions { , name = "CARDINALITY" , description = "Returns HyperLogLogPlus-estimated cardinality for this set. See [HLLP README](HLLP.md)" , params = {"hyperLogLogPlus - the hllp set"} - , returns = "Long value representing the cardinality for this set" + , returns = "Long value representing the cardinality for this set. Cardinality of a null set is 0." ) public static class HLLPCardinality extends BaseStellarFunction { @Override public Object apply(List<Object> args) { - if (args.size() < 1) { - throw new IllegalArgumentException("Must pass an hllp set to get the cardinality for"); + if (args.size() == 1) { + if (args.get(0) instanceof HyperLogLogPlus) { + HyperLogLogPlus hllpSet = (HyperLogLogPlus) args.get(0); + return hllpSet.cardinality(); + } else { + return 0L; + } + } else { + return 0L; } - return ((HyperLogLogPlus) args.get(0)).cardinality(); } } @@ -113,7 +119,7 @@ public class HyperLogLogPlusFunctions { , name = "MERGE" , description = "Merge hllp sets together. The resulting estimator is initialized with p and sp precision values from the first provided hllp estimator set. See [HLLP README](HLLP.md)" , params = {"hllp - List of hllp estimators to merge. Takes a single hllp set or a list."} - , returns = "A new merged HyperLogLogPlus estimator set" + , returns = "A new merged HyperLogLogPlus estimator set. Passing an empty list returns null." ) public static class HLLPMerge extends BaseStellarFunction { @@ -128,8 +134,8 @@ public class HyperLogLogPlusFunctions { } else { estimators.add(args.get(0)); } - if (estimators.size() < 1) { - throw new IllegalArgumentException("Must pass 1..n hllp sets to merge"); + if (estimators.size() == 0) { + return null; } HyperLogLogPlus hllp = ConversionUtils.convert(estimators.get(0), HyperLogLogPlus.class); if (estimators.size() > 1) { http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/345eba8d/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsIntegrationTest.java b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsIntegrationTest.java index 85e938a..cd9cae4 100644 --- a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsIntegrationTest.java +++ b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsIntegrationTest.java @@ -34,6 +34,7 @@ public class HyperLogLogPlusFunctionsIntegrationTest { put("val2", "miklavcic"); put("val3", "metron"); put("val4", "batman"); + put("nullArg", null); }}; /** @@ -117,4 +118,16 @@ public class HyperLogLogPlusFunctionsIntegrationTest { Assert.assertThat("Incorrect cardinality returned", estimate, equalTo(4L)); } + /** + *HLLP_CARDINALITY(nullArg) + */ + @Multiline + private static String zeroCardinalityRule; + + @Test + public void cardinality_of_null_value_is_0() { + Long estimate = (Long) StellarProcessorUtils.run(zeroCardinalityRule, values); + Assert.assertThat("Incorrect cardinality returned", estimate, equalTo(0L)); + } + } http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/345eba8d/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsTest.java ---------------------------------------------------------------------- diff --git a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsTest.java b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsTest.java index 20be8e4..a43eeb7 100644 --- a/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsTest.java +++ b/metron-analytics/metron-statistics/src/test/java/org/apache/metron/statistics/approximation/HyperLogLogPlusFunctionsTest.java @@ -23,7 +23,9 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import static org.hamcrest.CoreMatchers.*; @@ -84,10 +86,11 @@ public class HyperLogLogPlusFunctionsTest { } @Test - public void hllp_cardinality_with_invalid_arguments_throws_exception() { - thrown.expect(IllegalArgumentException.class); - thrown.expectMessage("Must pass an hllp set to get the cardinality for"); - new HyperLogLogPlusFunctions.HLLPCardinality().apply(ImmutableList.of()); + public void hllp_cardinality_returns_0_for_null_set() { + List nullArg = new ArrayList() {{ + add(null); + }}; + Assert.assertThat("Cardinality should be 0", new HyperLogLogPlusFunctions.HLLPCardinality().apply(nullArg), equalTo(0L)); } @Test @@ -139,4 +142,10 @@ public class HyperLogLogPlusFunctionsTest { new HyperLogLogPlusFunctions.HLLPMerge().apply(ImmutableList.of(hllp1, hllp2)); } + @Test + public void merge_returns_null_if_passed_an_empty_list_to_merge() { + List emptyList = ImmutableList.of(); + Assert.assertThat("Should be empty list", new HyperLogLogPlusFunctions.HLLPMerge().apply(ImmutableList.of(emptyList)), equalTo(null)); + } + }
