This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new e1b39e2318c [feature](lance) push down common string and boolean
predicates (#67051)
e1b39e2318c is described below
commit e1b39e2318ca58361e00986ea6bc705e07cc8bda
Author: jay <[email protected]>
AuthorDate: Sun Sep 6 06:15:18 2026 +0800
[feature](lance) push down common string and boolean predicates (#67051)
### What problem does this PR solve?
Lance regular scans currently leave common string predicates and direct
boolean predicates in Doris, so Lance cannot filter rows before Arrow
materialization and transfer.
### What is changed?
- Push two-argument `LIKE` / `NOT LIKE` predicates without backslash
escapes through the existing Substrait filter.
- Push `starts_with(column, literal)` and `ends_with(column, literal)`.
- Push direct boolean-column predicates, including `NOT boolean_column`.
- Keep explicit/custom LIKE escapes, REGEXP, non-literal patterns, and
non-string inputs as Doris residual predicates.
- Extend FE unit and external Lance regression coverage.
### Testing
- `mvn -pl fe-core -am -Dtest=LancePredicateConverterTest
-DfailIfNoTests=false test` (21 passed)
- `test_lance_scalar_predicate_pushdown` against local MinIO + FE + BE
- Manual end-to-end LargeUtf8 `starts_with` verification
### Release note
Improve Lance predicate pushdown for common string and boolean
expressions.
---
.../lance/source/LancePredicateConverter.java | 77 ++++++++++++++++++
.../lance/LancePredicateConverterTest.java | 90 ++++++++++++++++++++++
.../lance/test_lance_scalar_predicate_pushdown.out | 55 +++++++++++++
.../test_lance_scalar_predicate_pushdown.groovy | 68 +++++++++++++++-
4 files changed, 289 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LancePredicateConverter.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LancePredicateConverter.java
index 784896eff09..41fc1d855f8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LancePredicateConverter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LancePredicateConverter.java
@@ -24,14 +24,17 @@ import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.analysis.DecimalLiteral;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FloatLiteral;
+import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.InPredicate;
import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.LargeIntLiteral;
+import org.apache.doris.analysis.LikePredicate;
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.NullLiteral;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StringLiteral;
+import org.apache.doris.thrift.TFunctionBinaryType;
import io.substrait.expression.Expression;
import io.substrait.expression.ExpressionCreator;
@@ -131,6 +134,15 @@ public class LancePredicateConverter {
if (expr instanceof IsNullPredicate) {
return convertIsNull((IsNullPredicate) expr);
}
+ if (expr instanceof LikePredicate) {
+ return convertLike((LikePredicate) expr);
+ }
+ if (expr instanceof FunctionCallExpr) {
+ return convertStringFunction((FunctionCallExpr) expr);
+ }
+ if (expr instanceof SlotRef) {
+ return convertBooleanSlot((SlotRef) expr);
+ }
return Optional.empty();
}
@@ -257,6 +269,63 @@ public class LancePredicateConverter {
return Optional.of(comparisonFunction(function,
fieldReference(field)));
}
+ private Optional<Expression> convertLike(LikePredicate predicate) {
+ if (predicate.getOp() != LikePredicate.Operator.LIKE) {
+ return Optional.empty();
+ }
+ return convertStringPredicate("like:str_str", predicate.getChild(0),
predicate.getChild(1), true);
+ }
+
+ private Optional<Expression> convertStringFunction(FunctionCallExpr
function) {
+ if (function.getFnName() == null || function.getFn() == null
+ || function.getFn().getBinaryType() !=
TFunctionBinaryType.BUILTIN
+ || function.getChildren().size() != 2) {
+ return Optional.empty();
+ }
+ String functionName =
function.getFnName().getFunction().toLowerCase(Locale.ROOT);
+ switch (functionName) {
+ case "like":
+ return convertStringPredicate(
+ "like:str_str", function.getChild(0),
function.getChild(1), true);
+ case "starts_with":
+ return convertStringPredicate(
+ "starts_with:str_str", function.getChild(0),
function.getChild(1), false);
+ case "ends_with":
+ return convertStringPredicate(
+ "ends_with:str_str", function.getChild(0),
function.getChild(1), false);
+ default:
+ return Optional.empty();
+ }
+ }
+
+ private Optional<Expression> convertStringPredicate(
+ String function, Expr input, Expr pattern, boolean
rejectEscapedPattern) {
+ SlotRef slot = directSlot(input);
+ LiteralExpr literal = directLiteral(pattern);
+ ResolvedField field = slot == null ? null : findField(slot);
+ if (field == null || !isStringType(field.field.getType()) || !(literal
instanceof StringLiteral)) {
+ return Optional.empty();
+ }
+ String patternValue = literal.getStringValue();
+ // Doris uses backslash as LIKE's default escape character, while the
Substrait function
+ // has no escape argument. Keep escaped LIKE patterns in Doris rather
than changing meaning.
+ if (patternValue.indexOf('\0') >= 0
+ || (rejectEscapedPattern && patternValue.indexOf('\\') >= 0)) {
+ return Optional.empty();
+ }
+ return Optional.of(stringFunction(function, fieldReference(field),
+ ExpressionCreator.string(false, patternValue)));
+ }
+
+ private Optional<Expression> convertBooleanSlot(SlotRef slot) {
+ ResolvedField field = findField(slot);
+ if (field == null || !(field.field.getType() instanceof
ArrowType.Bool)) {
+ return Optional.empty();
+ }
+ return Optional.of(comparisonFunction("equal:any_any",
+ fieldReference(field), ExpressionCreator.bool(false, true)));
+ }
+
// convert doris literal to Substrait literal with arrow type
private Optional<Expression> convertLiteral(ArrowType type, LiteralExpr
literal) {
if (type instanceof ArrowType.Bool && literal instanceof BoolLiteral) {
@@ -450,6 +519,10 @@ public class LancePredicateConverter {
|| type instanceof ArrowType.LargeUtf8;
}
+ private static boolean isStringType(ArrowType type) {
+ return type instanceof ArrowType.Utf8 || type instanceof
ArrowType.LargeUtf8;
+ }
+
// slotref with ordinal index with Substrait Type
private Expression fieldReference(ResolvedField field) {
return FieldReference.newRootStructReference(field.ordinal,
toSubstraitType(field.field));
@@ -502,6 +575,10 @@ public class LancePredicateConverter {
return scalarFunction(DefaultExtensionCatalog.FUNCTIONS_BOOLEAN, key,
arguments);
}
+ private static Expression stringFunction(String key, Expression...
arguments) {
+ return scalarFunction(DefaultExtensionCatalog.FUNCTIONS_STRING, key,
Arrays.asList(arguments));
+ }
+
private static Expression scalarFunction(String uri, String key,
List<Expression> arguments) {
SimpleExtension.ScalarFunctionVariant declaration =
EXTENSIONS.getScalarFunction(
SimpleExtension.FunctionAnchor.of(uri, key));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LancePredicateConverterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LancePredicateConverterTest.java
index 66709546786..64e6fd70724 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LancePredicateConverterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LancePredicateConverterTest.java
@@ -23,15 +23,19 @@ import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.analysis.DecimalLiteral;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FunctionCallExpr;
+import org.apache.doris.analysis.FunctionName;
import org.apache.doris.analysis.InPredicate;
import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.LargeIntLiteral;
+import org.apache.doris.analysis.LikePredicate;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StringLiteral;
+import org.apache.doris.catalog.ScalarFunction;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.datasource.lance.source.LancePredicateConverter;
+import org.apache.doris.thrift.TFunctionBinaryType;
import io.substrait.proto.ExtendedExpression;
import org.apache.arrow.vector.types.DateUnit;
@@ -215,6 +219,85 @@ public class LancePredicateConverterTest {
Assertions.assertEquals(3, result.getPushedConjuncts().size());
}
+ @Test
+ public void testStringPredicates() {
+ Expr legacyLike = new LikePredicate(LikePredicate.Operator.LIKE,
+ new SlotRef(null, "label"), new StringLiteral("ready%"));
+ Expr like = stringFunction(
+ "like", new SlotRef(null, "label"), new
StringLiteral("%ead_"));
+ Expr startsWith = stringFunction(
+ "starts_with", new SlotRef(null, "label"), new
StringLiteral("ready"));
+ Expr endsWith = stringFunction(
+ "ends_with", new SlotRef(null, "large_label"), new
StringLiteral("done"));
+
+ LancePredicateConverter.ConversionResult result =
+ converter.convert(Arrays.asList(legacyLike, like, startsWith,
endsWith));
+
+ ExtendedExpression envelope = Assertions.assertDoesNotThrow(
+ () ->
ExtendedExpression.parseFrom(result.getSubstraitFilter()));
+ String serialized = envelope.toString();
+ Assertions.assertTrue(serialized.contains("like:str_str"));
+ Assertions.assertTrue(serialized.contains("starts_with:str_str"));
+ Assertions.assertTrue(serialized.contains("ends_with:str_str"));
+ Assertions.assertEquals(4, result.getPushedConjuncts().size());
+ }
+
+ @Test
+ public void testUnsupportedStringPredicatesRemainResidual() {
+ Expr regexp = new LikePredicate(LikePredicate.Operator.REGEXP,
+ new SlotRef(null, "label"), new StringLiteral("ready.*"));
+ Expr escapedLike = new LikePredicate(LikePredicate.Operator.LIKE,
+ new SlotRef(null, "label"), new StringLiteral("ready\\%"));
+ Expr explicitEscape = stringFunction("like",
+ new SlotRef(null, "label"), new StringLiteral("ready!%"), new
StringLiteral("!"));
+ Expr nonLiteralPattern = stringFunction(
+ "starts_with", new SlotRef(null, "label"), new SlotRef(null,
"event-type"));
+ Expr nonStringInput = stringFunction(
+ "ends_with", new SlotRef(null, "row_id"), new
StringLiteral("1"));
+
+ LancePredicateConverter.ConversionResult result = converter.convert(
+ Arrays.asList(regexp, escapedLike, explicitEscape,
nonLiteralPattern, nonStringInput));
+
+ Assertions.assertEquals(0, result.getSubstraitFilter().length);
+ Assertions.assertTrue(result.getPushedConjuncts().isEmpty());
+ }
+
+ @Test
+ public void testResolvedUdfAndNulStringPredicatesRemainResidual() {
+ FunctionCallExpr udf = stringFunction(
+ "starts_with", new SlotRef(null, "label"), new
StringLiteral("ready"));
+ udf.getFn().setBinaryType(TFunctionBinaryType.JAVA_UDF);
+ Expr legacyNulLike = new LikePredicate(LikePredicate.Operator.LIKE,
+ new SlotRef(null, "label"), new StringLiteral("m\0_"));
+ Expr functionNulLike = stringFunction(
+ "like", new SlotRef(null, "label"), new StringLiteral("m\0_"));
+
+ LancePredicateConverter.ConversionResult result =
+ converter.convert(Arrays.asList(udf, legacyNulLike,
functionNulLike));
+
+ Assertions.assertEquals(0, result.getSubstraitFilter().length);
+ Assertions.assertTrue(result.getPushedConjuncts().isEmpty());
+ }
+
+ @Test
+ public void testDirectBooleanPredicates() {
+ LancePredicateConverter boolConverter = new
LancePredicateConverter(new Schema(
+ Collections.singletonList(Field.nullable("active",
ArrowType.Bool.INSTANCE))));
+ Expr active = new SlotRef(null, "active");
+ Expr notActive = new CompoundPredicate(
+ CompoundPredicate.Operator.NOT, new SlotRef(null, "active"),
null);
+
+ LancePredicateConverter.ConversionResult result =
+ boolConverter.convert(Arrays.asList(active, notActive));
+
+ ExtendedExpression envelope = Assertions.assertDoesNotThrow(
+ () ->
ExtendedExpression.parseFrom(result.getSubstraitFilter()));
+ String serialized = envelope.toString();
+ Assertions.assertTrue(serialized.contains("equal:any_any"));
+ Assertions.assertTrue(serialized.contains("not:bool"));
+ Assertions.assertEquals(2, result.getPushedConjuncts().size());
+ }
+
@Test
public void testNullableNullSafeEqualityPreservesTwoValuedSemantics() {
Expr nullableNullSafeEqual = new
BinaryPredicate(BinaryPredicate.Operator.EQ_FOR_NULL,
@@ -473,6 +556,13 @@ public class LancePredicateConverterTest {
Assertions.assertEquals(1, result.getPushedConjuncts().size());
}
+ private FunctionCallExpr stringFunction(String name, Expr... arguments) {
+ FunctionCallExpr function = new FunctionCallExpr(name,
Arrays.asList(arguments));
+ function.setFn(new ScalarFunction(new FunctionName(name),
+ Collections.nCopies(arguments.length, Type.VARCHAR),
Type.BOOLEAN, false, true));
+ return function;
+ }
+
private void assertNullSafeEqualityComposition(Expr predicate) {
LancePredicateConverter.ConversionResult result =
converter.convert(Collections.singletonList(predicate));
diff --git
a/regression-test/data/external_table_p0/lance/test_lance_scalar_predicate_pushdown.out
b/regression-test/data/external_table_p0/lance/test_lance_scalar_predicate_pushdown.out
index 91b378709d6..595e35993dd 100644
---
a/regression-test/data/external_table_p0/lance/test_lance_scalar_predicate_pushdown.out
+++
b/regression-test/data/external_table_p0/lance/test_lance_scalar_predicate_pushdown.out
@@ -78,6 +78,61 @@
9
10
+-- !select_bool_direct --
+5
+6
+7
+9
+10
+
+-- !select_bool_direct_not --
+2
+3
+4
+8
+
+-- !select_utf8_starts_with --
+7
+8
+
+-- !select_utf8_ends_with --
+4
+6
+
+-- !select_utf8_like_prefix --
+2
+4
+10
+
+-- !select_utf8_like_contains --
+3
+7
+10
+
+-- !select_utf8_like_single_wildcard --
+7
+8
+
+-- !select_utf8_not_like --
+2
+3
+4
+5
+6
+9
+10
+
+-- !select_utf8_like_explicit_escape --
+
+-- !select_utf8_like_nul_residual --
+2
+4
+10
+
+-- !select_utf8_regexp_residual --
+7
+8
+
-- !select_float32_eq --
7
8
diff --git
a/regression-test/suites/external_table_p0/lance/test_lance_scalar_predicate_pushdown.groovy
b/regression-test/suites/external_table_p0/lance/test_lance_scalar_predicate_pushdown.groovy
index edd47dff285..f52f37cdfaa 100644
---
a/regression-test/suites/external_table_p0/lance/test_lance_scalar_predicate_pushdown.groovy
+++
b/regression-test/suites/external_table_p0/lance/test_lance_scalar_predicate_pushdown.groovy
@@ -22,7 +22,7 @@ suite("test_lance_scalar_predicate_pushdown", "p0,external") {
*
* | Lance / Arrow type | Doris type | Operators exercised |
* |---|---|---|
- * | bool | boolean | =, !=, <>, <=>, IN, NOT IN, IS NULL, IS NOT NULL,
OR, NOT |
+ * | bool | boolean | Direct predicate, =, !=, <>, <=>, IN, NOT IN, IS
NULL, IS NOT NULL, OR, NOT |
* | float32 | float | All operators below |
* | float64 | double | All operators below |
* | decimal128 | decimal(18,2) | All operators below |
@@ -84,6 +84,19 @@ suite("test_lance_scalar_predicate_pushdown", "p0,external")
{
}
}
+ Closure verifyResidual = { String query, String expression ->
+ explain {
+ sql(query)
+ notContains "lancePushdownPredicate="
+ check { explainString ->
+ String residual = explainString.readLines()
+ .find { line ->
line.trim().startsWith("predicates:") }
+ return residual != null
+ &&
residual.toLowerCase().contains(expression.toLowerCase())
+ }
+ }
+ }
+
Closure verifyOrderedScalarPushdown = { String tableName, String
typeName, String columnName, Map values ->
String eqQuery = """ SELECT row_id FROM ${tableName} WHERE
${columnName} = ${values.equal} ORDER BY row_id; """
verifyFullyPushedDown(eqQuery, columnName)
@@ -202,10 +215,63 @@ suite("test_lance_scalar_predicate_pushdown",
"p0,external") {
String boolReversedQuery = """ SELECT row_id FROM
predicate_pushdown WHERE true = bool_value ORDER BY row_id; """
verifyFullyPushedDown(boolReversedQuery, "bool_value")
quickTest("select_bool_reversed", boolReversedQuery)
+
+ String boolDirectQuery = """ SELECT row_id FROM predicate_pushdown
WHERE bool_value ORDER BY row_id; """
+ verifyFullyPushedDown(boolDirectQuery, "bool_value")
+ quickTest("select_bool_direct", boolDirectQuery)
+
+ String boolDirectNotQuery = """ SELECT row_id FROM
predicate_pushdown WHERE NOT bool_value ORDER BY row_id; """
+ verifyFullyPushedDown(boolDirectNotQuery, "bool_value")
+ quickTest("select_bool_direct_not", boolDirectNotQuery)
}
verifyBooleanPushdown()
+ String startsWithQuery =
+ """ SELECT row_id FROM predicate_pushdown WHERE
starts_with(utf8_value, 'ten') ORDER BY row_id; """
+ verifyFullyPushedDown(startsWithQuery, "utf8_value")
+ quickTest("select_utf8_starts_with", startsWithQuery)
+
+ String endsWithQuery =
+ """ SELECT row_id FROM predicate_pushdown WHERE
ends_with(utf8_value, 'one') ORDER BY row_id; """
+ verifyFullyPushedDown(endsWithQuery, "utf8_value")
+ quickTest("select_utf8_ends_with", endsWithQuery)
+
+ String likePrefixQuery =
+ """ SELECT row_id FROM predicate_pushdown WHERE utf8_value
LIKE 'm%' ORDER BY row_id; """
+ verifyFullyPushedDown(likePrefixQuery, "utf8_value")
+ quickTest("select_utf8_like_prefix", likePrefixQuery)
+
+ String likeContainsQuery =
+ """ SELECT row_id FROM predicate_pushdown WHERE utf8_value
LIKE '%a%' ORDER BY row_id; """
+ verifyFullyPushedDown(likeContainsQuery, "utf8_value")
+ quickTest("select_utf8_like_contains", likeContainsQuery)
+
+ String likeSingleWildcardQuery =
+ """ SELECT row_id FROM predicate_pushdown WHERE utf8_value
LIKE 'ten-_' ORDER BY row_id; """
+ verifyFullyPushedDown(likeSingleWildcardQuery, "utf8_value")
+ quickTest("select_utf8_like_single_wildcard", likeSingleWildcardQuery)
+
+ String notLikeQuery =
+ """ SELECT row_id FROM predicate_pushdown WHERE utf8_value NOT
LIKE 'ten-%' ORDER BY row_id; """
+ verifyFullyPushedDown(notLikeQuery, "utf8_value")
+ quickTest("select_utf8_not_like", notLikeQuery)
+
+ String explicitEscapeQuery =
+ """ SELECT row_id FROM predicate_pushdown WHERE utf8_value
LIKE 'ten!_%' ESCAPE '!' ORDER BY row_id; """
+ verifyResidual(explicitEscapeQuery, "like")
+ quickTest("select_utf8_like_explicit_escape", explicitEscapeQuery)
+
+ String nulLikeQuery =
+ """ SELECT row_id FROM predicate_pushdown WHERE utf8_value
LIKE 'm\\0_' ORDER BY row_id; """
+ verifyResidual(nulLikeQuery, "like")
+ quickTest("select_utf8_like_nul_residual", nulLikeQuery)
+
+ String regexpQuery =
+ """ SELECT row_id FROM predicate_pushdown WHERE utf8_value
REGEXP '^ten-' ORDER BY row_id; """
+ verifyResidual(regexpQuery, "regexp")
+ quickTest("select_utf8_regexp_residual", regexpQuery)
+
verifyOrderedScalarPushdown("predicate_pushdown", "float32",
"float32_value", [
equal: "10",
threshold: "0",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
