This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 2cfb00b1de Add missing `isNull()` implementation to
`FilteredAggregator` (#14465)
2cfb00b1de is described below
commit 2cfb00b1de928a48a1c61bc04c10cdbccab33866
Author: Abhishek Radhakrishnan <[email protected]>
AuthorDate: Tue Jun 27 16:35:15 2023 -0700
Add missing `isNull()` implementation to `FilteredAggregator` (#14465)
---
.../query/aggregation/FilteredAggregator.java | 6 +
.../query/aggregation/FilteredAggregatorTest.java | 396 ++++++++++++---------
.../segment/TestNullableDoubleColumnSelector.java | 4 -
.../segment/TestNullableFloatColumnSelector.java | 4 -
.../segment/TestNullableLongColumnSelector.java | 4 -
.../segment/WrappingDimensionSelectorTest.java | 3 +-
6 files changed, 226 insertions(+), 191 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregator.java
b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregator.java
index 6ffea5b70f..edb6b8a601 100644
---
a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregator.java
+++
b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregator.java
@@ -40,6 +40,12 @@ public class FilteredAggregator implements Aggregator
}
}
+ @Override
+ public boolean isNull()
+ {
+ return delegate.isNull();
+ }
+
@Override
public Object get()
{
diff --git
a/processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorTest.java
b/processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorTest.java
index 7c1d809594..da2e7be83d 100644
---
a/processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorTest.java
+++
b/processing/src/test/java/org/apache/druid/query/aggregation/FilteredAggregatorTest.java
@@ -45,46 +45,221 @@ import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.DimensionSelector;
import org.apache.druid.segment.DimensionSelectorUtils;
import org.apache.druid.segment.IdLookup;
+import org.apache.druid.segment.TestNullableFloatColumnSelector;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.data.IndexedInts;
import org.apache.druid.segment.data.SingleIndexedInt;
+import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
import org.junit.Test;
import javax.annotation.Nullable;
import java.util.Arrays;
-public class FilteredAggregatorTest
+public class FilteredAggregatorTest extends InitializedNullHandlingTest
{
- private void aggregate(TestFloatColumnSelector selector, FilteredAggregator
agg)
+ @Test
+ public void testAggregate()
{
- agg.aggregate();
- selector.increment();
+ final Float[] values = {0.15f, 0.27f};
+ final TestNullableFloatColumnSelector selector = new
TestNullableFloatColumnSelector(values);
+ final FilteredAggregatorFactory factory = new FilteredAggregatorFactory(
+ new FloatSumAggregatorFactory("billy", "value"),
+ new SelectorDimFilter("dim", "a", null)
+ );
+
+ final Float[] expectedVals = {values[0], values[0] + values[1]};
+ validateFilteredAggs(factory, selector, expectedVals);
}
@Test
- public void testAggregate()
+ public void testAggregateWithNullVals()
{
- final float[] values = {0.15f, 0.27f};
- final TestFloatColumnSelector selector = new
TestFloatColumnSelector(values);
+ final Float[] values = {0.15f, null, 0.27f};
+ final TestNullableFloatColumnSelector selector = new
TestNullableFloatColumnSelector(values);
- FilteredAggregatorFactory factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
+ final FilteredAggregatorFactory factory = new FilteredAggregatorFactory(
+ new FloatSumAggregatorFactory("billy", "value"),
+ new SelectorDimFilter("dim", "a", null)
+ );
+
+ final Float[] expectedValues = {values[0], values[0], values[0] +
values[2]};
+ validateFilteredAggs(factory, selector, expectedValues);
+ }
+
+ @Test
+ public void testAggregateOnlyWithNulls()
+ {
+ final Float[] values = {null, null};
+ final TestNullableFloatColumnSelector selector = new
TestNullableFloatColumnSelector(values);
+ final FilteredAggregatorFactory factory = new FilteredAggregatorFactory(
+ new FloatSumAggregatorFactory("billy", "value"),
new SelectorDimFilter("dim", "a", null)
);
- FilteredAggregator agg = (FilteredAggregator)
factory.factorize(makeColumnSelector(selector));
+ final Float[] expectedValues = {NullHandling.defaultFloatValue(),
NullHandling.defaultFloatValue()};
+ validateFilteredAggs(factory, selector, expectedValues);
+ }
+
+
+ @Test
+ public void testAggregateWithNotFilter()
+ {
+ final Float[] values = {0.15f, 0.27f};
+ final TestNullableFloatColumnSelector selector = new
TestNullableFloatColumnSelector(values);
+ final FilteredAggregatorFactory factory = new FilteredAggregatorFactory(
+ new DoubleSumAggregatorFactory("billy", "value"),
+ new NotDimFilter(new SelectorDimFilter("dim", "b", null))
+ );
+
+ final Float[] expectedValues = {values[0], values[0] + values[1]};
+ validateFilteredAggs(factory, selector, expectedValues);
+ }
+
+ @Test
+ public void testAggregateWithOrFilter()
+ {
+ final Float[] values = {0.15f, 0.27f, 0.14f};
+ final TestNullableFloatColumnSelector selector = new
TestNullableFloatColumnSelector(values);
+
+ final FilteredAggregatorFactory factory = new FilteredAggregatorFactory(
+ new DoubleSumAggregatorFactory("billy", "value"),
+ new OrDimFilter(Lists.newArrayList(
+ new SelectorDimFilter("dim", "a", null),
+ new SelectorDimFilter("dim", "b", null)
+ ))
+ );
+
+ final Float[] expectedValues = {values[0], values[0] + values[1],
values[0] + values[1] + values[2]};
+ validateFilteredAggs(factory, selector, expectedValues);
+ }
+
+ @Test
+ public void testAggregateWithAndFilter()
+ {
+ final Float[] values = {0.15f, 0.27f};
+ final TestNullableFloatColumnSelector selector = new
TestNullableFloatColumnSelector(values);
+ final FilteredAggregatorFactory factory = new FilteredAggregatorFactory(
+ new DoubleSumAggregatorFactory("billy", "value"),
+ new AndDimFilter(Lists.newArrayList(
+ new NotDimFilter(new SelectorDimFilter("dim", "b", null)),
+ new SelectorDimFilter("dim", "a", null)
+ ))
+ );
+
+ final Float[] expectedValues = {values[0], values[0] + values[1]};
+ validateFilteredAggs(factory, selector, expectedValues);
+ }
+
+ @Test
+ public void testAggregateWithPredicateFilters2()
+ {
+ final Float[] values = {0.15f, 0.27f, null};
+ final Float[] expectedValues = {values[0], values[0] + values[1],
values[0] + values[1]};
+ TestNullableFloatColumnSelector selector;
+ FilteredAggregatorFactory factory;
+
+ factory = new FilteredAggregatorFactory(
+ new DoubleSumAggregatorFactory("billy", "value"),
+ new BoundDimFilter("dim", "a", "a", false, false, true, null,
StringComparators.ALPHANUMERIC)
+ );
+ selector = new TestNullableFloatColumnSelector(values);
+ validateFilteredAggs(factory, selector, expectedValues);
+
+ factory = new FilteredAggregatorFactory(
+ new DoubleSumAggregatorFactory("billy", "value"),
+ new RegexDimFilter("dim", "a", null)
+ );
+ selector = new TestNullableFloatColumnSelector(values);
+ validateFilteredAggs(factory, selector, expectedValues);
- double expectedFirst = new Float(values[0]).doubleValue();
- double expectedSecond = new Float(values[1]).doubleValue() + expectedFirst;
- double expectedThird = expectedSecond;
+ factory = new FilteredAggregatorFactory(
+ new DoubleSumAggregatorFactory("billy", "value"),
+ new SearchQueryDimFilter("dim", new ContainsSearchQuerySpec("a",
true), null)
+ );
+ selector = new TestNullableFloatColumnSelector(values);
+ validateFilteredAggs(factory, selector, expectedValues);
- assertValues(agg, selector, expectedFirst, expectedSecond, expectedThird);
+ String jsFn = "function(x) { return(x === 'a') }";
+ factory = new FilteredAggregatorFactory(
+ new DoubleSumAggregatorFactory("billy", "value"),
+ new JavaScriptDimFilter("dim", jsFn, null,
JavaScriptConfig.getEnabledInstance())
+ );
+ selector = new TestNullableFloatColumnSelector(values);
+ validateFilteredAggs(factory, selector, expectedValues);
}
- private ColumnSelectorFactory makeColumnSelector(final
TestFloatColumnSelector selector)
+ @Test
+ public void testAggregateWithExtractionFns()
+ {
+ final Float[] values = {0.15f, null, 0.27f, null, 0.13f};
+ final Float[] expectedValues = {
+ values[0],
+ values[0],
+ values[0] + values[2],
+ values[0] + values[2],
+ values[0] + values[2] + values[4]
+ };
+
+ TestNullableFloatColumnSelector selector;
+ FilteredAggregatorFactory factory;
+
+ String extractionJsFn = "function(str) { return str + 'AARDVARK'; }";
+ ExtractionFn extractionFn = new JavaScriptExtractionFn(
+ extractionJsFn,
+ false,
+ JavaScriptConfig.getEnabledInstance()
+ );
+
+ factory = new FilteredAggregatorFactory(
+ new DoubleSumAggregatorFactory("billy", "value"),
+ new SelectorDimFilter("dim", "aAARDVARK", extractionFn)
+ );
+ selector = new TestNullableFloatColumnSelector(values);
+ validateFilteredAggs(factory, selector, expectedValues);
+
+ factory = new FilteredAggregatorFactory(
+ new DoubleSumAggregatorFactory("billy", "value"),
+ new InDimFilter("dim", Arrays.asList("NOT-aAARDVARK", "FOOBAR",
"aAARDVARK"), extractionFn)
+ );
+ selector = new TestNullableFloatColumnSelector(values);
+ validateFilteredAggs(factory, selector, expectedValues);
+
+ factory = new FilteredAggregatorFactory(
+ new DoubleSumAggregatorFactory("billy", "value"),
+ new BoundDimFilter("dim", "aAARDVARK", "aAARDVARK", false, false,
true, extractionFn,
+ StringComparators.ALPHANUMERIC
+ )
+ );
+ selector = new TestNullableFloatColumnSelector(values);
+ validateFilteredAggs(factory, selector, expectedValues);
+
+ factory = new FilteredAggregatorFactory(
+ new DoubleSumAggregatorFactory("billy", "value"),
+ new RegexDimFilter("dim", "aAARDVARK", extractionFn)
+ );
+ selector = new TestNullableFloatColumnSelector(values);
+ validateFilteredAggs(factory, selector, expectedValues);
+
+ factory = new FilteredAggregatorFactory(
+ new DoubleSumAggregatorFactory("billy", "value"),
+ new SearchQueryDimFilter("dim", new
ContainsSearchQuerySpec("aAARDVARK", true), extractionFn)
+ );
+ selector = new TestNullableFloatColumnSelector(values);
+ validateFilteredAggs(factory, selector, expectedValues);
+
+ String jsFn = "function(x) { return(x === 'aAARDVARK') }";
+ factory = new FilteredAggregatorFactory(
+ new DoubleSumAggregatorFactory("billy", "value"),
+ new JavaScriptDimFilter("dim", jsFn, extractionFn,
JavaScriptConfig.getEnabledInstance())
+ );
+ selector = new TestNullableFloatColumnSelector(values);
+ validateFilteredAggs(factory, selector, expectedValues);
+ }
+
+ private ColumnSelectorFactory makeColumnSelector(final
TestNullableFloatColumnSelector selector)
{
return new ColumnSelectorFactory()
@@ -102,11 +277,7 @@ public class FilteredAggregatorTest
public IndexedInts getRow()
{
SingleIndexedInt row = new SingleIndexedInt();
- if (selector.getIndex() % 3 == 2) {
- row.setValue(1);
- } else {
- row.setValue(0);
- }
+ row.setValue(0);
return row;
}
@@ -216,174 +387,43 @@ public class FilteredAggregatorTest
};
}
- private void assertValues(FilteredAggregator agg, TestFloatColumnSelector
selector, double... expectedVals)
- {
- Assert.assertEquals(NullHandling.defaultDoubleValue(), agg.get());
- Assert.assertEquals(NullHandling.defaultDoubleValue(), agg.get());
- Assert.assertEquals(NullHandling.defaultDoubleValue(), agg.get());
- for (double expectedVal : expectedVals) {
- aggregate(selector, agg);
- Assert.assertEquals(expectedVal, agg.get());
- Assert.assertEquals(expectedVal, agg.get());
- Assert.assertEquals(expectedVal, agg.get());
- }
- }
-
- @Test
- public void testAggregateWithNotFilter()
- {
- final float[] values = {0.15f, 0.27f};
- final TestFloatColumnSelector selector = new
TestFloatColumnSelector(values);
-
- FilteredAggregatorFactory factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
- new NotDimFilter(new SelectorDimFilter("dim", "b", null))
- );
-
- validateFilteredAggs(factory, values, selector);
- }
-
- @Test
- public void testAggregateWithOrFilter()
- {
- final float[] values = {0.15f, 0.27f, 0.14f};
- final TestFloatColumnSelector selector = new
TestFloatColumnSelector(values);
-
- FilteredAggregatorFactory factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
- new OrDimFilter(Lists.newArrayList(new SelectorDimFilter("dim", "a",
null), new SelectorDimFilter("dim", "b", null)))
- );
-
- FilteredAggregator agg = (FilteredAggregator) factory.factorize(
- makeColumnSelector(selector)
- );
-
- double expectedFirst = new Float(values[0]).doubleValue();
- double expectedSecond = new Float(values[1]).doubleValue() + expectedFirst;
- double expectedThird = expectedSecond + new Float(values[2]).doubleValue();
- assertValues(agg, selector, expectedFirst, expectedSecond, expectedThird);
- }
-
- @Test
- public void testAggregateWithAndFilter()
- {
- final float[] values = {0.15f, 0.27f};
- final TestFloatColumnSelector selector = new
TestFloatColumnSelector(values);
-
- FilteredAggregatorFactory factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
- new AndDimFilter(Lists.newArrayList(new NotDimFilter(new
SelectorDimFilter("dim", "b", null)), new SelectorDimFilter("dim", "a",
null))));
-
- validateFilteredAggs(factory, values, selector);
- }
-
- @Test
- public void testAggregateWithPredicateFilters()
- {
- final float[] values = {0.15f, 0.27f};
- TestFloatColumnSelector selector;
- FilteredAggregatorFactory factory;
-
- factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
- new BoundDimFilter("dim", "a", "a", false, false, true, null,
StringComparators.ALPHANUMERIC)
- );
- selector = new TestFloatColumnSelector(values);
- validateFilteredAggs(factory, values, selector);
-
- factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
- new RegexDimFilter("dim", "a", null)
- );
- selector = new TestFloatColumnSelector(values);
- validateFilteredAggs(factory, values, selector);
-
- factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
- new SearchQueryDimFilter("dim", new ContainsSearchQuerySpec("a",
true), null)
- );
- selector = new TestFloatColumnSelector(values);
- validateFilteredAggs(factory, values, selector);
-
- String jsFn = "function(x) { return(x === 'a') }";
- factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
- new JavaScriptDimFilter("dim", jsFn, null,
JavaScriptConfig.getEnabledInstance())
- );
- selector = new TestFloatColumnSelector(values);
- validateFilteredAggs(factory, values, selector);
- }
-
- @Test
- public void testAggregateWithExtractionFns()
+ private void aggregate(
+ final TestNullableFloatColumnSelector selector,
+ final FilteredAggregator agg
+ )
{
- final float[] values = {0.15f, 0.27f};
- TestFloatColumnSelector selector;
- FilteredAggregatorFactory factory;
-
- String extractionJsFn = "function(str) { return str + 'AARDVARK'; }";
- ExtractionFn extractionFn = new JavaScriptExtractionFn(extractionJsFn,
false, JavaScriptConfig.getEnabledInstance());
-
- factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
- new SelectorDimFilter("dim", "aAARDVARK", extractionFn)
- );
- selector = new TestFloatColumnSelector(values);
- validateFilteredAggs(factory, values, selector);
-
- factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
- new InDimFilter("dim", Arrays.asList("NOT-aAARDVARK", "FOOBAR",
"aAARDVARK"), extractionFn)
- );
- selector = new TestFloatColumnSelector(values);
- validateFilteredAggs(factory, values, selector);
-
- factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
- new BoundDimFilter("dim", "aAARDVARK", "aAARDVARK", false, false,
true, extractionFn,
- StringComparators.ALPHANUMERIC
- )
- );
- selector = new TestFloatColumnSelector(values);
- validateFilteredAggs(factory, values, selector);
-
- factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
- new RegexDimFilter("dim", "aAARDVARK", extractionFn)
- );
- selector = new TestFloatColumnSelector(values);
- validateFilteredAggs(factory, values, selector);
-
- factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
- new SearchQueryDimFilter("dim", new
ContainsSearchQuerySpec("aAARDVARK", true), extractionFn)
- );
- selector = new TestFloatColumnSelector(values);
- validateFilteredAggs(factory, values, selector);
-
- String jsFn = "function(x) { return(x === 'aAARDVARK') }";
- factory = new FilteredAggregatorFactory(
- new DoubleSumAggregatorFactory("billy", "value"),
- new JavaScriptDimFilter("dim", jsFn, extractionFn,
JavaScriptConfig.getEnabledInstance())
- );
- selector = new TestFloatColumnSelector(values);
- validateFilteredAggs(factory, values, selector);
+ agg.aggregate();
+ selector.increment();
}
private void validateFilteredAggs(
- FilteredAggregatorFactory factory,
- float[] values,
- TestFloatColumnSelector selector
+ final FilteredAggregatorFactory factory,
+ final TestNullableFloatColumnSelector selector,
+ final Float[] expectedValues
)
{
FilteredAggregator agg = (FilteredAggregator) factory.factorize(
makeColumnSelector(selector)
);
- double expectedFirst = new Float(values[0]).doubleValue();
- double expectedSecond = new Float(values[1]).doubleValue() + expectedFirst;
- double expectedThird = expectedSecond;
+ // Validate state before any aggregation
+ if (NullHandling.sqlCompatible()) {
+ Assert.assertTrue(agg.isNull());
+ Assert.assertNull(agg.get());
+ } else {
+ Assert.assertFalse(agg.isNull());
+ Assert.assertEquals(0.0f, agg.getFloat(), 0.001);
+ }
- assertValues(agg, selector, expectedFirst, expectedSecond, expectedThird);
+ for (Float expectedValue : expectedValues) {
+ aggregate(selector, agg);
+ if (expectedValue == null) {
+ Assert.assertTrue(agg.isNull());
+ Assert.assertNull(agg.get());
+ } else {
+ Assert.assertFalse(agg.isNull());
+ Assert.assertEquals(expectedValue, agg.getFloat(), 0.001);
+ }
+ }
}
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/TestNullableDoubleColumnSelector.java
b/processing/src/test/java/org/apache/druid/segment/TestNullableDoubleColumnSelector.java
index 609ce1e01d..3953070127 100644
---
a/processing/src/test/java/org/apache/druid/segment/TestNullableDoubleColumnSelector.java
+++
b/processing/src/test/java/org/apache/druid/segment/TestNullableDoubleColumnSelector.java
@@ -25,10 +25,6 @@ public class TestNullableDoubleColumnSelector extends
TestDoubleColumnSelector
{
private final Double[] doubles;
- static {
- NullHandling.initializeForTests();
- }
-
private int index = 0;
public TestNullableDoubleColumnSelector(Double[] doubles)
diff --git
a/processing/src/test/java/org/apache/druid/segment/TestNullableFloatColumnSelector.java
b/processing/src/test/java/org/apache/druid/segment/TestNullableFloatColumnSelector.java
index 19b49610de..a96b31cb92 100644
---
a/processing/src/test/java/org/apache/druid/segment/TestNullableFloatColumnSelector.java
+++
b/processing/src/test/java/org/apache/druid/segment/TestNullableFloatColumnSelector.java
@@ -26,10 +26,6 @@ public class TestNullableFloatColumnSelector extends
TestFloatColumnSelector
private final Float[] floats;
- static {
- NullHandling.initializeForTests();
- }
-
private int index = 0;
public TestNullableFloatColumnSelector(Float[] floats)
diff --git
a/processing/src/test/java/org/apache/druid/segment/TestNullableLongColumnSelector.java
b/processing/src/test/java/org/apache/druid/segment/TestNullableLongColumnSelector.java
index 5e27e995c7..a30b6c9e35 100644
---
a/processing/src/test/java/org/apache/druid/segment/TestNullableLongColumnSelector.java
+++
b/processing/src/test/java/org/apache/druid/segment/TestNullableLongColumnSelector.java
@@ -25,10 +25,6 @@ public class TestNullableLongColumnSelector extends
TestLongColumnSelector
{
private final Long[] longs;
- static {
- NullHandling.initializeForTests();
- }
-
private int index = 0;
public TestNullableLongColumnSelector(Long[] longs)
diff --git
a/processing/src/test/java/org/apache/druid/segment/WrappingDimensionSelectorTest.java
b/processing/src/test/java/org/apache/druid/segment/WrappingDimensionSelectorTest.java
index b208b96f81..8ba3425807 100644
---
a/processing/src/test/java/org/apache/druid/segment/WrappingDimensionSelectorTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/WrappingDimensionSelectorTest.java
@@ -20,10 +20,11 @@
package org.apache.druid.segment;
import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
import org.junit.Test;
-public class WrappingDimensionSelectorTest
+public class WrappingDimensionSelectorTest extends InitializedNullHandlingTest
{
@Test
public void testLongWrappingDimensionSelector()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
