This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 709fe0a7b04 [fix](nereids) Do not infer MATCH and SEARCH predicates
across column equalities (#69419)
709fe0a7b04 is described below
commit 709fe0a7b04264cd537f43de89ce1a3ca183b8c9
Author: Jack <[email protected]>
AuthorDate: Fri Oct 23 10:56:27 2026 +0800
[fix](nereids) Do not infer MATCH and SEARCH predicates across column
equalities (#69419)
Related PR: #69292 (where review found it)
Problem Summary:
Predicate inference copies a predicate on one column to every column
that is equal to it. When the predicate contains MATCH, the copy changes
the result:
```sql
CREATE TABLE t (id INT, c1 TEXT, c2 TEXT,
INDEX i1 (c1) USING INVERTED PROPERTIES ("parser" = "english"),
INDEX i2 (c2) USING INVERTED)
DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES
("replication_num" = "1");
INSERT INTO t VALUES (1, 'apple red', 'apple red'), (2, 'banana', 'banana');
SELECT id FROM t WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY
'pear');
-- no rows; row 1 is expected
```
`InferPredicateByReplace` added `(c2 MATCH_ANY 'apple' OR c2 MATCH_ANY
'pear')` to the scan. A MATCH depends on the inverted index of its
column: `c2` has no parser, so a MATCH on `c2` only matches a whole
value, and the copy dropped row 1. The same happens when the MATCH is
inside a comparison, e.g. `CAST((c1 MATCH_ANY 'apple' OR c1 MATCH_ANY
'pear') AS INT) > 0`, and with `multi_match(c1, 'any', 'apple')` and
`search()`, which also run through the inverted index of their column.
When both columns have the same index, the copy is scored too. With
`'apple red'` and `'apple green'` in both columns, `score()` of the same
filter with `ORDER BY score() DESC LIMIT 10` doubles from 0.4700036 to
0.9400072. Adding `score() > 0.6` returns both rows, although their
score is below the minimum.
`InferPredicateByReplace.infer` now skips every input that contains
`Match`, `MultiMatch`, `Search` or `SearchExpression`, whatever it is
nested in, next to its existing `NoneMovableFunction` and
non-deterministic checks. Their result depends on the inverted index of
their column, not only on its value, so their column must not be
replaced by an equal one. Other predicates next to them are still
inferred.
---------
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../rules/rewrite/InferPredicateByReplace.java | 8 ++-
.../rules/rewrite/InferPredicateByReplaceTest.java | 54 ++++++++++++++++
.../infer_predicate/infer_match_predicate.out | 19 ++++++
.../infer_predicate/infer_match_predicate.groovy | 75 ++++++++++++++++++++++
4 files changed, 155 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java
index 9003b802476..757e7c75b08 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java
@@ -28,11 +28,15 @@ import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.Like;
+import org.apache.doris.nereids.trees.expressions.Match;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Or;
+import org.apache.doris.nereids.trees.expressions.SearchExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
import
org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MultiMatch;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Search;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -237,7 +241,9 @@ public class InferPredicateByReplace {
Map<Expression, Set<Expression>> exprPredicates = new HashMap<>();
for (Expression input : inputs) {
// Inference can evaluate a predicate on rows that never reach its
original filter.
- if (input.anyMatch(expr -> expr instanceof NoneMovableFunction
+ // MATCH, multi_match and SEARCH depend on column indexes, and
copies can change score().
+ if (input.containsType(Match.class, MultiMatch.class,
Search.class, SearchExpression.class)
+ || input.anyMatch(expr -> expr instanceof
NoneMovableFunction
|| !((ExpressionTrait) expr).isDeterministic())
|| input.getInputSlots().size() != 1) {
continue;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java
index f9bd8835bda..ad5afddd9f1 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.rules.rewrite;
+import org.apache.doris.analysis.SearchDslParser;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
@@ -24,15 +25,21 @@ import
org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.Like;
+import org.apache.doris.nereids.trees.expressions.MatchAny;
+import org.apache.doris.nereids.trees.expressions.MatchPhrase;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Or;
+import org.apache.doris.nereids.trees.expressions.SearchExpression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Abs;
import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue;
import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Length;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.MultiMatch;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Search;
import org.apache.doris.nereids.trees.expressions.functions.scalar.SignBit;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
@@ -90,6 +97,53 @@ public class InferPredicateByReplaceTest {
Assertions.assertEquals(inputs,
PredicateInferUtils.inferAllPredicate(inputs));
}
+ @ParameterizedTest
+ @MethodSource("matchAndSearchPredicates")
+ public void testDoNotInferMatchOrSearchPredicates(Expression predicate,
SlotReference a, SlotReference b) {
+ Set<Expression> inputs = new LinkedHashSet<>(ImmutableList.of(new
EqualTo(a, b), predicate));
+
+ Assertions.assertEquals(inputs, InferPredicateByReplace.infer(inputs));
+ Assertions.assertEquals(inputs,
PredicateInferUtils.inferPredicate(inputs));
+ Assertions.assertEquals(inputs,
PredicateInferUtils.inferAllPredicate(inputs));
+ }
+
+ private static Stream<Arguments> matchAndSearchPredicates() {
+ SlotReference a = new SlotReference("a", StringType.INSTANCE);
+ SlotReference b = new SlotReference("b", StringType.INSTANCE);
+ return Stream.of(
+ new MatchAny(a, new StringLiteral("apple")),
+ new MatchPhrase(a, new StringLiteral("apple red")),
+ new MultiMatch(a, new StringLiteral("any"), new
StringLiteral("apple")),
+ new Search(new StringLiteral("a:apple")),
+ new SearchExpression("a:apple",
SearchDslParser.parseDsl("a:apple"), ImmutableList.of(a)))
+ .flatMap(search -> {
+ Expression or = new Or(search, new EqualTo(a, new
StringLiteral("pear")));
+ return Stream.of(or,
+ new EqualTo(search, BooleanLiteral.TRUE),
+ new EqualTo(or, BooleanLiteral.TRUE),
+ new GreaterThan(new Cast(or,
IntegerType.INSTANCE), new IntegerLiteral(0)),
+ new InPredicate(or,
ImmutableList.of(BooleanLiteral.TRUE)),
+ new Not(new EqualTo(or, BooleanLiteral.TRUE)),
+ new Like(new Cast(or, StringType.INSTANCE), new
StringLiteral("%true%")));
+ })
+ .map(predicate -> Arguments.of(predicate, a, b));
+ }
+
+ @Test
+ public void testInferOrdinaryPredicatesAlongsideMatch() {
+ SlotReference a = new SlotReference("a", StringType.INSTANCE);
+ SlotReference b = new SlotReference("b", StringType.INSTANCE);
+ Expression match = new Or(new MatchAny(a, new StringLiteral("apple")),
+ new MatchAny(a, new StringLiteral("pear")));
+ Set<Expression> inputs = new LinkedHashSet<>(ImmutableList.of(
+ new EqualTo(a, b), match, new Like(a, new
StringLiteral("apple%"))));
+ Set<Expression> expected = new LinkedHashSet<>(inputs);
+ expected.add(new Like(b, new StringLiteral("apple%")));
+
+ Assertions.assertEquals(expected,
InferPredicateByReplace.infer(inputs));
+ Assertions.assertEquals(expected,
PredicateInferUtils.inferAllPredicate(inputs));
+ }
+
@Test
public void testInferWithEqualTo() {
SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
diff --git
a/regression-test/data/nereids_rules_p0/infer_predicate/infer_match_predicate.out
b/regression-test/data/nereids_rules_p0/infer_predicate/infer_match_predicate.out
new file mode 100644
index 00000000000..cdc045bdd81
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/infer_predicate/infer_match_predicate.out
@@ -0,0 +1,19 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !different_index --
+1
+
+-- !wrapped_match --
+1
+
+-- !search --
+1
+
+-- !multi_match --
+1
+
+-- !score --
+1 0.4700036
+2 0.4700036
+
+-- !minimum_score --
+
diff --git
a/regression-test/suites/nereids_rules_p0/infer_predicate/infer_match_predicate.groovy
b/regression-test/suites/nereids_rules_p0/infer_predicate/infer_match_predicate.groovy
new file mode 100644
index 00000000000..6cdb841dae8
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/infer_predicate/infer_match_predicate.groovy
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// An equality between two columns does not let a MATCH on one column be
copied to the other:
+// MATCH depends on the inverted index of its column, and a copy would add to
score().
+suite("infer_match_predicate") {
+ // c2 has an index without a parser, so MATCH on c2 only matches a whole
value.
+ sql "DROP TABLE IF EXISTS infer_match_predicate_parser"
+ sql """
+ CREATE TABLE infer_match_predicate_parser (
+ id INT,
+ c1 TEXT,
+ c2 TEXT,
+ INDEX idx_c1 (c1) USING INVERTED PROPERTIES ("parser" = "english"),
+ INDEX idx_c2 (c2) USING INVERTED
+ ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql "INSERT INTO infer_match_predicate_parser VALUES (1, 'apple red',
'apple red'), (2, 'banana', 'banana')"
+ order_qt_different_index """
+ SELECT id FROM infer_match_predicate_parser
+ WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear')
+ """
+ order_qt_wrapped_match """
+ SELECT id FROM infer_match_predicate_parser
+ WHERE c1 = c2 AND CAST((c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear')
AS INT) > 0
+ """
+ order_qt_search """
+ SELECT id FROM infer_match_predicate_parser
+ WHERE c1 = c2 AND (search('c1:apple') OR c1 = 'pear')
+ """
+ order_qt_multi_match """
+ SELECT id FROM infer_match_predicate_parser
+ WHERE c1 = c2 AND (multi_match(c1, 'any', 'apple') OR multi_match(c1,
'any', 'pear'))
+ """
+
+ sql "DROP TABLE IF EXISTS infer_match_predicate_score"
+ sql """
+ CREATE TABLE infer_match_predicate_score (
+ id INT,
+ c1 TEXT,
+ c2 TEXT,
+ INDEX idx_c1 (c1) USING INVERTED PROPERTIES ("parser" = "english"),
+ INDEX idx_c2 (c2) USING INVERTED PROPERTIES ("parser" = "english")
+ ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO infer_match_predicate_score VALUES
+ (1, 'apple red', 'apple red'), (2, 'apple green', 'apple green'),
(3, 'banana yellow', 'banana yellow')
+ """
+ order_qt_score """
+ SELECT id, score() AS s FROM infer_match_predicate_score
+ WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear') ORDER
BY s DESC LIMIT 10
+ """
+ order_qt_minimum_score """
+ SELECT id, score() AS s FROM infer_match_predicate_score
+ WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear') AND
score() > 0.6
+ ORDER BY s DESC LIMIT 10
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
