This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 480a947e8b6 Fail projection filter match early if any child in
`AndFilter` failed rewrite (#18407)
480a947e8b6 is described below
commit 480a947e8b6afa6f2637954928fa82da15c05037
Author: Cece Mei <[email protected]>
AuthorDate: Tue Aug 26 23:31:28 2025 +0200
Fail projection filter match early if any child in `AndFilter` failed
rewrite (#18407)
---
.../segment/projections/ProjectionFilterMatch.java | 91 ++++++++++++++-
.../druid/segment/projections/Projections.java | 85 +-------------
.../projections/ProjectionFilterMatchTest.java | 130 +++++++++++++++++++++
.../druid/segment/projections/ProjectionsTest.java | 64 ----------
4 files changed, 220 insertions(+), 150 deletions(-)
diff --git
a/processing/src/main/java/org/apache/druid/segment/projections/ProjectionFilterMatch.java
b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionFilterMatch.java
index ab4b79624bd..0b5dfce7bdf 100644
---
a/processing/src/main/java/org/apache/druid/segment/projections/ProjectionFilterMatch.java
+++
b/processing/src/main/java/org/apache/druid/segment/projections/ProjectionFilterMatch.java
@@ -19,12 +19,17 @@
package org.apache.druid.segment.projections;
+import com.google.common.collect.Lists;
import org.apache.druid.query.filter.Filter;
+import org.apache.druid.segment.filter.AndFilter;
+import org.apache.druid.segment.filter.IsBooleanFilter;
import org.apache.druid.segment.filter.TrueFilter;
+import javax.annotation.Nullable;
+import java.util.List;
+
/**
- * Terminal marker used by {@link Projections#rewriteFilter(Filter, Filter)}
indicating that a {@link Filter} can be
- * dropped because it matches the filter of a projection
+ * Terminal marker indicating that a {@link Filter} can be dropped because it
matches the filter of a projection
*/
public final class ProjectionFilterMatch extends TrueFilter
{
@@ -34,4 +39,86 @@ public final class ProjectionFilterMatch extends TrueFilter
{
// no instantiation
}
+
+ /**
+ * Rewrites a query {@link Filter} if possible, removing the {@link Filter}
of a projection. To match a projection
+ * filter, the query filter must be equal to the projection filter, or must
contain the projection filter as the child
+ * of an AND filter. This method returns null
+ * indicating that a rewrite is impossible with the implication that the
query cannot use the projection because the
+ * projection doesn't contain all the rows the query would match if not
using the projection.
+ */
+ @Nullable
+ public static Filter rewriteFilter(Filter projectionFilter, Filter
queryFilter)
+ {
+ if (projectionFilter.equals(queryFilter)) {
+ return ProjectionFilterMatch.INSTANCE;
+ }
+ if (queryFilter instanceof IsBooleanFilter && ((IsBooleanFilter)
queryFilter).isTrue()) {
+ final IsBooleanFilter isTrueFilter = (IsBooleanFilter) queryFilter;
+ final Filter rewritten = rewriteFilter(projectionFilter,
isTrueFilter.getBaseFilter());
+ if (rewritten == null) {
+ return null;
+ }
+ //noinspection ObjectEquality
+ if (rewritten == ProjectionFilterMatch.INSTANCE) {
+ return ProjectionFilterMatch.INSTANCE;
+ }
+ return new IsBooleanFilter(rewritten, true);
+ }
+ if (queryFilter instanceof AndFilter) {
+ AndFilter andFilter = (AndFilter) queryFilter;
+
+ // if both and filters, check to see if the query and filter contains
all of the clauses of the projection and filter
+ if (projectionFilter instanceof AndFilter) {
+ AndFilter projectionAndFilter = (AndFilter) projectionFilter;
+ Filter rewritten = andFilter;
+ // calling rewriteFilter using each child of the projection AND filter
as the projection filter will remove
+ // the child from the query AND filter if it exists (or return null if
it does not exist, since it must exist
+ // to be a valid rewrite). The remaining AND filter of will only
contain children that were not part of the
+ // projection AND filter
+ for (Filter filter : projectionAndFilter.getFilters()) {
+ rewritten = rewriteFilter(filter, rewritten);
+ if (rewritten != null) {
+ if (rewritten == ProjectionFilterMatch.INSTANCE) {
+ return ProjectionFilterMatch.INSTANCE;
+ }
+ } else {
+ return null;
+ }
+ }
+ if (rewritten != null) {
+ return rewritten;
+ }
+ return null;
+ }
+
+ // else check to see if any clause of the query AND filter is the
projection filter
+ List<Filter> newChildren =
Lists.newArrayListWithExpectedSize(andFilter.getFilters().size());
+ boolean childRewritten = false;
+ for (Filter filter : andFilter.getFilters()) {
+ Filter rewritten = rewriteFilter(projectionFilter, filter);
+ //noinspection ObjectEquality
+ if (rewritten == ProjectionFilterMatch.INSTANCE) {
+ childRewritten = true;
+ } else {
+ if (rewritten != null) {
+ newChildren.add(rewritten);
+ childRewritten = true;
+ } else {
+ newChildren.add(filter);
+ }
+ }
+ }
+ // at least one child must have been rewritten to rewrite the AND
+ if (childRewritten) {
+ if (newChildren.size() > 1) {
+ return new AndFilter(newChildren);
+ } else {
+ return newChildren.get(0);
+ }
+ }
+ return null;
+ }
+ return null;
+ }
}
diff --git
a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java
b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java
index 47258886017..7b280544abc 100644
---
a/processing/src/main/java/org/apache/druid/segment/projections/Projections.java
+++
b/processing/src/main/java/org/apache/druid/segment/projections/Projections.java
@@ -19,7 +19,6 @@
package org.apache.druid.segment.projections;
-import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.druid.data.input.impl.AggregateProjectionSpec;
import org.apache.druid.error.InvalidInput;
@@ -33,8 +32,6 @@ import org.apache.druid.segment.CursorBuildSpec;
import org.apache.druid.segment.CursorHolder;
import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.segment.column.ColumnHolder;
-import org.apache.druid.segment.filter.AndFilter;
-import org.apache.druid.segment.filter.IsBooleanFilter;
import org.apache.druid.utils.CollectionUtils;
import javax.annotation.Nullable;
@@ -198,7 +195,7 @@ public class Projections
final Filter remappedQueryFilter =
queryFilter.rewriteRequiredColumns(filterRewrites);
- final Filter rewritten = rewriteFilter(projectionFilter,
remappedQueryFilter);
+ final Filter rewritten =
ProjectionFilterMatch.rewriteFilter(projectionFilter, remappedQueryFilter);
// if the filter does not contain the projection filter, we cannot
match this projection
if (rewritten == null) {
return null;
@@ -472,86 +469,6 @@ public class Projections
return null;
}
- /**
- * Rewrites a query {@link Filter} if possible, removing the {@link Filter}
of a projection. To match a projection
- * filter, the query filter must be equal to the projection filter, or must
contain the projection filter as the child
- * of an AND filter. This method returns null
- * indicating that a rewrite is impossible with the implication that the
query cannot use the projection because the
- * projection doesn't contain all the rows the query would match if not
using the projection.
- */
- @Nullable
- public static Filter rewriteFilter(Filter projectionFilter, Filter
queryFilter)
- {
- if (queryFilter.equals(projectionFilter)) {
- return ProjectionFilterMatch.INSTANCE;
- }
- if (queryFilter instanceof IsBooleanFilter && ((IsBooleanFilter)
queryFilter).isTrue()) {
- final IsBooleanFilter isTrueFilter = (IsBooleanFilter) queryFilter;
- final Filter rewritten = rewriteFilter(projectionFilter,
isTrueFilter.getBaseFilter());
- if (rewritten == null) {
- return null;
- }
- //noinspection ObjectEquality
- if (rewritten == ProjectionFilterMatch.INSTANCE) {
- return ProjectionFilterMatch.INSTANCE;
- }
- return new IsBooleanFilter(rewritten, true);
- }
- if (queryFilter instanceof AndFilter) {
- AndFilter andFilter = (AndFilter) queryFilter;
-
- // if both and filters, check to see if the query and filter contains
all of the clauses of the projection and filter
- if (projectionFilter instanceof AndFilter) {
- AndFilter projectionAndFilter = (AndFilter) projectionFilter;
- Filter rewritten = andFilter;
- // calling rewriteFilter using each child of the projection AND filter
as the projection filter will remove
- // the child from the query AND filter if it exists (or return null if
it does not exist, since it must exist
- // to be a valid rewrite). The remaining AND filter of will only
contain children that were not part of the
- // projection AND filter
- for (Filter filter : projectionAndFilter.getFilters()) {
- rewritten = rewriteFilter(filter, rewritten);
- if (rewritten != null) {
- if (rewritten == ProjectionFilterMatch.INSTANCE) {
- return ProjectionFilterMatch.INSTANCE;
- }
- }
- }
- if (rewritten != null) {
- return rewritten;
- }
- return null;
- }
-
- // else check to see if any clause of the query AND filter is the
projection filter
- List<Filter> newChildren =
Lists.newArrayListWithExpectedSize(andFilter.getFilters().size());
- boolean childRewritten = false;
- for (Filter filter : andFilter.getFilters()) {
- Filter rewritten = rewriteFilter(projectionFilter, filter);
- //noinspection ObjectEquality
- if (rewritten == ProjectionFilterMatch.INSTANCE) {
- childRewritten = true;
- } else {
- if (rewritten != null) {
- newChildren.add(rewritten);
- childRewritten = true;
- } else {
- newChildren.add(filter);
- }
- }
- }
- // at least one child must have been rewritten to rewrite the AND
- if (childRewritten) {
- if (newChildren.size() > 1) {
- return new AndFilter(newChildren);
- } else {
- return newChildren.get(0);
- }
- }
- return null;
- }
- return null;
- }
-
public static String
getProjectionSmooshV9FileName(AggregateProjectionMetadata projectionSpec,
String columnName)
{
return getProjectionSmooshV9Prefix(projectionSpec) + columnName;
diff --git
a/processing/src/test/java/org/apache/druid/segment/projections/ProjectionFilterMatchTest.java
b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionFilterMatchTest.java
new file mode 100644
index 00000000000..91494a7608b
--- /dev/null
+++
b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionFilterMatchTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.projections;
+
+import org.apache.druid.query.filter.EqualityFilter;
+import org.apache.druid.query.filter.Filter;
+import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.filter.AndFilter;
+import org.apache.druid.segment.filter.IsBooleanFilter;
+import org.apache.druid.segment.filter.OrFilter;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+public class ProjectionFilterMatchTest
+{
+ @Test
+ void testRewriteFilter()
+ {
+ Filter xeqfoo = new EqualityFilter("x", ColumnType.STRING, "foo", null);
+ Filter xeqfoo2 = new EqualityFilter("x", ColumnType.STRING, "foo", null);
+ Filter xeqbar = new EqualityFilter("x", ColumnType.STRING, "bar", null);
+ Filter yeqbar = new EqualityFilter("y", ColumnType.STRING, "bar", null);
+ Filter zeq123 = new EqualityFilter("z", ColumnType.LONG, 123L, null);
+
+ Filter queryFilter = xeqfoo2;
+ Assertions.assertInstanceOf(
+ ProjectionFilterMatch.class,
+ ProjectionFilterMatch.rewriteFilter(xeqfoo, queryFilter)
+ );
+
+ queryFilter = yeqbar;
+ Assertions.assertNull(ProjectionFilterMatch.rewriteFilter(xeqfoo,
queryFilter));
+
+ queryFilter = new AndFilter(List.of(xeqfoo, yeqbar));
+ Assertions.assertEquals(
+ yeqbar,
+ ProjectionFilterMatch.rewriteFilter(xeqfoo, queryFilter)
+ );
+
+ queryFilter = new AndFilter(List.of(new OrFilter(List.of(xeqfoo, xeqbar)),
yeqbar));
+ Assertions.assertNull(ProjectionFilterMatch.rewriteFilter(xeqfoo,
queryFilter));
+
+ queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, true),
yeqbar));
+ Assertions.assertEquals(yeqbar,
ProjectionFilterMatch.rewriteFilter(xeqfoo, queryFilter));
+
+ queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, false),
yeqbar));
+ Assertions.assertNull(ProjectionFilterMatch.rewriteFilter(xeqfoo,
queryFilter));
+
+ queryFilter = new AndFilter(List.of(new AndFilter(List.of(xeqfoo,
yeqbar)), zeq123));
+ Assertions.assertEquals(
+ new AndFilter(List.of(yeqbar, zeq123)),
+ ProjectionFilterMatch.rewriteFilter(xeqfoo, queryFilter)
+ );
+
+ queryFilter = new AndFilter(
+ List.of(
+ new EqualityFilter("a", ColumnType.STRING, "foo", null),
+ new EqualityFilter("b", ColumnType.STRING, "bar", null),
+ new EqualityFilter("c", ColumnType.STRING, "baz", null)
+ )
+ );
+ Assertions.assertEquals(
+ new EqualityFilter("b", ColumnType.STRING, "bar", null),
+ ProjectionFilterMatch.rewriteFilter(
+ new AndFilter(
+ List.of(
+ new EqualityFilter("a", ColumnType.STRING, "foo", null),
+ new EqualityFilter("c", ColumnType.STRING, "baz", null)
+ )
+ ),
+ queryFilter
+ )
+ );
+ }
+
+ @Test
+ void testRewriteFilter_andFilterNotMatch()
+ {
+ Filter xeqfoo = new EqualityFilter("x", ColumnType.STRING, "foo", null);
+ Filter metricLike = new EqualityFilter("metric", ColumnType.STRING,
"metric1", null);
+ Filter envOrUser = new OrFilter(List.of(
+ new EqualityFilter("env", ColumnType.STRING, "dev", null),
+ new EqualityFilter("user", ColumnType.STRING, "user-a", null)
+ ));
+ Filter envOrUser2 = new OrFilter(List.of(
+ new EqualityFilter("prod", ColumnType.STRING, "dev", null),
+ new EqualityFilter("another-user", ColumnType.STRING, "user-a", null)
+ ));
+
+ Filter queryFilter = new AndFilter(List.of(xeqfoo, envOrUser, metricLike));
+ Filter projectionFilter = new AndFilter(List.of(envOrUser2, metricLike));
+
+
Assertions.assertNull(ProjectionFilterMatch.rewriteFilter(projectionFilter,
queryFilter));
+ }
+
+ @Test
+ void testRewriteFilter_andFilterMatch()
+ {
+ Filter xeqfoo = new EqualityFilter("x", ColumnType.STRING, "foo", null);
+ Filter metricLike = new EqualityFilter("metric", ColumnType.STRING,
"metric1", null);
+ Filter envOrUser = new OrFilter(List.of(
+ new EqualityFilter("env", ColumnType.STRING, "dev", null),
+ new EqualityFilter("user", ColumnType.STRING, "user-a", null)
+ ));
+
+ Filter queryFilter = new AndFilter(List.of(xeqfoo, envOrUser, metricLike));
+ Filter projectionFilter = new AndFilter(List.of(envOrUser, metricLike));
+
+ Assertions.assertEquals(xeqfoo,
ProjectionFilterMatch.rewriteFilter(projectionFilter, queryFilter));
+ }
+}
diff --git
a/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java
b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java
index 3c13a0571bf..dc86fb8ab2f 100644
---
a/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/projections/ProjectionsTest.java
@@ -24,15 +24,11 @@ import org.apache.druid.data.input.impl.LongDimensionSchema;
import org.apache.druid.data.input.impl.StringDimensionSchema;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
import org.apache.druid.query.filter.EqualityFilter;
-import org.apache.druid.query.filter.Filter;
import org.apache.druid.query.filter.LikeDimFilter;
import org.apache.druid.segment.AggregateProjectionMetadata;
import org.apache.druid.segment.CursorBuildSpec;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
-import org.apache.druid.segment.filter.AndFilter;
-import org.apache.druid.segment.filter.IsBooleanFilter;
-import org.apache.druid.segment.filter.OrFilter;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -219,66 +215,6 @@ class ProjectionsTest
Assertions.assertEquals(expected, projectionMatch);
}
- @Test
- void testRewriteFilter()
- {
- Filter xeqfoo = new EqualityFilter("x", ColumnType.STRING, "foo", null);
- Filter xeqfoo2 = new EqualityFilter("x", ColumnType.STRING, "foo", null);
- Filter xeqbar = new EqualityFilter("x", ColumnType.STRING, "bar", null);
- Filter yeqbar = new EqualityFilter("y", ColumnType.STRING, "bar", null);
- Filter zeq123 = new EqualityFilter("z", ColumnType.LONG, 123L, null);
-
- Filter queryFilter = xeqfoo2;
- Assertions.assertInstanceOf(
- ProjectionFilterMatch.class,
- Projections.rewriteFilter(xeqfoo, queryFilter)
- );
-
- queryFilter = yeqbar;
- Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter));
-
- queryFilter = new AndFilter(List.of(xeqfoo, yeqbar));
- Assertions.assertEquals(
- yeqbar,
- Projections.rewriteFilter(xeqfoo, queryFilter)
- );
-
- queryFilter = new AndFilter(List.of(new OrFilter(List.of(xeqfoo, xeqbar)),
yeqbar));
- Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter));
-
- queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, true),
yeqbar));
- Assertions.assertEquals(yeqbar, Projections.rewriteFilter(xeqfoo,
queryFilter));
-
- queryFilter = new AndFilter(List.of(new IsBooleanFilter(xeqfoo, false),
yeqbar));
- Assertions.assertNull(Projections.rewriteFilter(xeqfoo, queryFilter));
-
- queryFilter = new AndFilter(List.of(new AndFilter(List.of(xeqfoo,
yeqbar)), zeq123));
- Assertions.assertEquals(
- new AndFilter(List.of(yeqbar, zeq123)),
- Projections.rewriteFilter(xeqfoo, queryFilter)
- );
-
- queryFilter = new AndFilter(
- List.of(
- new EqualityFilter("a", ColumnType.STRING, "foo", null),
- new EqualityFilter("b", ColumnType.STRING, "bar", null),
- new EqualityFilter("c", ColumnType.STRING, "baz", null)
- )
- );
- Assertions.assertEquals(
- new EqualityFilter("b", ColumnType.STRING, "bar", null),
- Projections.rewriteFilter(
- new AndFilter(
- List.of(
- new EqualityFilter("a", ColumnType.STRING, "foo", null),
- new EqualityFilter("c", ColumnType.STRING, "baz", null)
- )
- ),
- queryFilter
- )
- );
- }
-
private static class RowSignatureChecker implements
Projections.PhysicalColumnChecker
{
private final RowSignature rowSignature;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
