This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 2845b6a424 add new filters to unnest filter pushdown (#14777)
2845b6a424 is described below
commit 2845b6a424acc9b5cea93a95325c1c94c94aeb76
Author: Clint Wylie <[email protected]>
AuthorDate: Tue Aug 8 03:29:18 2023 -0700
add new filters to unnest filter pushdown (#14777)
---
.../apache/druid/segment/UnnestStorageAdapter.java | 12 ++++-
.../druid/segment/UnnestStorageAdapterTest.java | 58 ++++++++++++++++++++++
2 files changed, 68 insertions(+), 2 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
index 00a119388a..048cd10f8e 100644
---
a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
+++
b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
@@ -19,6 +19,7 @@
package org.apache.druid.segment;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.apache.druid.java.util.common.Pair;
@@ -28,8 +29,11 @@ import org.apache.druid.java.util.common.guava.Sequences;
import org.apache.druid.query.QueryMetrics;
import org.apache.druid.query.filter.BooleanFilter;
import org.apache.druid.query.filter.DimFilter;
+import org.apache.druid.query.filter.EqualityFilter;
import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.InDimFilter;
+import org.apache.druid.query.filter.NullFilter;
+import org.apache.druid.query.filter.RangeFilter;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.data.Indexed;
@@ -452,7 +456,8 @@ public class UnnestStorageAdapter implements StorageAdapter
* over multi-value strings. (Rather than treat them as arrays.) There isn't
a method on the Filter interface that
* tells us this, so resort to instanceof.
*/
- private static boolean filterMapsOverMultiValueStrings(final Filter filter)
+ @VisibleForTesting
+ static boolean filterMapsOverMultiValueStrings(final Filter filter)
{
if (filter instanceof BooleanFilter) {
for (Filter child : ((BooleanFilter) filter).getFilters()) {
@@ -468,7 +473,10 @@ public class UnnestStorageAdapter implements StorageAdapter
return filter instanceof SelectorFilter
|| filter instanceof InDimFilter
|| filter instanceof LikeFilter
- || filter instanceof BoundFilter;
+ || filter instanceof BoundFilter
+ || filter instanceof NullFilter
+ || filter instanceof EqualityFilter
+ || filter instanceof RangeFilter;
}
}
diff --git
a/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java
b/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java
index 757a60d59e..480c86f51f 100644
---
a/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java
@@ -20,6 +20,7 @@
package org.apache.druid.segment;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.granularity.Granularity;
@@ -28,11 +29,26 @@ import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.query.QueryMetrics;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
+import org.apache.druid.query.filter.BoundDimFilter;
+import org.apache.druid.query.filter.EqualityFilter;
import org.apache.druid.query.filter.Filter;
+import org.apache.druid.query.filter.InDimFilter;
+import org.apache.druid.query.filter.LikeDimFilter;
+import org.apache.druid.query.filter.NullFilter;
+import org.apache.druid.query.filter.RangeFilter;
import org.apache.druid.query.filter.SelectorDimFilter;
import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.ValueType;
+import org.apache.druid.segment.filter.AndFilter;
+import org.apache.druid.segment.filter.BoundFilter;
+import org.apache.druid.segment.filter.ColumnComparisonFilter;
+import org.apache.druid.segment.filter.FalseFilter;
+import org.apache.druid.segment.filter.LikeFilter;
+import org.apache.druid.segment.filter.NotFilter;
import org.apache.druid.segment.filter.OrFilter;
+import org.apache.druid.segment.filter.SelectorFilter;
+import org.apache.druid.segment.filter.TrueFilter;
import org.apache.druid.segment.generator.GeneratorBasicSchemas;
import org.apache.druid.segment.generator.GeneratorSchemaInfo;
import org.apache.druid.segment.generator.SegmentGenerator;
@@ -392,6 +408,48 @@ public class UnnestStorageAdapterTest extends
InitializedNullHandlingTest
return null;
});
}
+
+ @Test
+ public void testAllowedFiltersForPushdown()
+ {
+ Filter[] allowed = new Filter[] {
+ new SelectorFilter("column", "value"),
+ new InDimFilter("column", ImmutableSet.of("a", "b")),
+ new LikeFilter("column", null,
LikeDimFilter.LikeMatcher.from("hello%", null), null),
+ new BoundFilter(
+ new BoundDimFilter("column", "a", "b", true, true, null, null,
null)
+ ),
+ NullFilter.forColumn("column"),
+ new EqualityFilter("column", ColumnType.LONG, 1234L, null),
+ new RangeFilter("column", ColumnType.LONG, 0L, 1234L, true, false,
null)
+ };
+ // not exhaustive
+ Filter[] notAllowed = new Filter[] {
+ TrueFilter.instance(),
+ FalseFilter.instance(),
+ new
ColumnComparisonFilter(ImmutableList.of(DefaultDimensionSpec.of("col1"),
DefaultDimensionSpec.of("col2")))
+ };
+
+ for (Filter f : allowed) {
+
Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(f));
+ }
+ for (Filter f : notAllowed) {
+
Assert.assertFalse(UnnestStorageAdapter.filterMapsOverMultiValueStrings(f));
+ }
+
+ Filter notAnd = new NotFilter(
+ new AndFilter(
+ Arrays.asList(allowed)
+ )
+ );
+
+
Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(notAnd));
+ Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(new
OrFilter(Arrays.asList(allowed))));
+ Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(new
NotFilter(notAnd)));
+ Assert.assertFalse(
+ UnnestStorageAdapter.filterMapsOverMultiValueStrings(new NotFilter(new
OrFilter(Arrays.asList(notAllowed))))
+ );
+ }
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
