This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new c601f4428 IMPALA-14737 Part2: Add relaxed predicate pushdown for LIKE
patterns with suffix
c601f4428 is described below
commit c601f44281805e421d2ce401729a703e5b16345b
Author: Arnab Karmakar <[email protected]>
AuthorDate: Thu Feb 26 09:28:06 2026 -0800
IMPALA-14737 Part2: Add relaxed predicate pushdown for LIKE patterns with
suffix
Part 1 (committed as 540a3784e) added basic LIKE predicate pushdown
to Iceberg for simple prefix patterns ('abc%'), exact matches ('exact'),
and escaped wildcards ('asd\%'). Patterns with literal content after
wildcards (e.g., 'prefix%suffix', 'd%d') were rejected and not pushed
down at all.
This Part 2 patch enhances the implementation with "relaxed predicate
pushdown" for patterns with suffix. Instead of rejecting these patterns
completely, we now:
1. Push down a relaxed prefix predicate to Iceberg (e.g., startsWith(
'prefix')) for partition/file pruning
2. Retain the full LIKE predicate (e.g., LIKE 'prefix%suffix') in the
scan node for Impala to evaluate on the surviving rows
Additionally, this patch adds support for simple LIKE patterns in DROP
PARTITION and SHOW FILES operations. Previously, any LIKE predicate would
fail in these operations. Now:
- Simple prefix patterns (e.g., 's LIKE d%') work correctly
- Patterns with suffix are rejected with clear error messages to prevent
unintended data loss (e.g., DROP PARTITION with 'd%d' would incorrectly
drop all partitions starting with 'd')
This provides significant performance benefits by leveraging Iceberg's
metadata filtering while maintaining query correctness.
Example behavior for `SELECT ... WHERE s LIKE 'd%d'`:
- Before: Pattern rejected, all 3/3 partitions scanned, no pruning
benefit
- After: startsWith('d') pushed to Iceberg -> 1/3 partitions, full LIKE
'd%d' evaluated by Impala on surviving rows -> correct results
Testing:
- Updated iceberg-like-pushdown.test with relaxed predicate tests
- Updated DROP PARTITION tests to include relaxed predicate tests
- Updated SHOW FILES tests to include relaxed predicate tests
Change-Id: I97c11362f098507fa440eafde3c35bbc6d7092b3
Reviewed-on: http://gerrit.cloudera.org:8080/24045
Reviewed-by: Impala Public Jenkins <[email protected]>
Tested-by: Zoltan Borok-Nagy <[email protected]>
---
.../analysis/AlterTableDropPartitionStmt.java | 22 +-
.../IcebergPartitionExpressionRewriter.java | 22 +-
.../org/apache/impala/analysis/ShowFilesStmt.java | 22 +-
.../org/apache/impala/analysis/ShowStatsStmt.java | 18 +-
.../impala/common/IcebergPredicateConverter.java | 319 ++++++++++++++-------
.../apache/impala/planner/IcebergScanPlanner.java | 45 ++-
.../queries/PlannerTest/iceberg-predicates.test | 5 +-
.../queries/QueryTest/iceberg-drop-partition.test | 60 ++++
.../queries/QueryTest/iceberg-like-pushdown.test | 15 +-
.../QueryTest/iceberg-show-files-partition.test | 53 +++-
10 files changed, 449 insertions(+), 132 deletions(-)
diff --git
a/fe/src/main/java/org/apache/impala/analysis/AlterTableDropPartitionStmt.java
b/fe/src/main/java/org/apache/impala/analysis/AlterTableDropPartitionStmt.java
index 3f17b7045..28fb411bf 100644
---
a/fe/src/main/java/org/apache/impala/analysis/AlterTableDropPartitionStmt.java
+++
b/fe/src/main/java/org/apache/impala/analysis/AlterTableDropPartitionStmt.java
@@ -157,12 +157,24 @@ public class AlterTableDropPartitionStmt extends
AlterTableStmt {
expr = rewriter.rewrite(expr);
expr.analyze(analyzer);
analyzer.getConstantFolder().rewrite(expr, analyzer);
- try {
- icebergPartitionExprs.add(converter.convert(expr));
- } catch (ImpalaException e) {
- throw new AnalysisException(
- "Invalid partition filtering expression: " + expr.toSql());
+
+ IcebergPredicateConverter.ConverterResult result =
converter.convert(expr);
+
+ if (result.isFailed()) {
+ throw new AnalysisException(String.format(
+ "Invalid partition filtering expression: %s. %s",
+ expr.toSql(), result.getErrorMessage()));
}
+
+ if (result.isPartiallyConverted()) {
+ throw new AnalysisException(String.format(
+ "Predicate '%s' can only be partially converted to Iceberg
expression: " +
+ "'%s'. Partially converted predicates are not allowed in DROP
PARTITION as " +
+ "they could drop more partitions than intended. Use a fully
convertible " +
+ "predicate instead.", expr.toSql(),
result.getIcebergExpression()));
+ }
+
+ icebergPartitionExprs.add(result.getIcebergExpression());
}
try (CloseableIterable<FileScanTask> fileScanTasks =
IcebergUtil.planFiles(table,
diff --git
a/fe/src/main/java/org/apache/impala/analysis/IcebergPartitionExpressionRewriter.java
b/fe/src/main/java/org/apache/impala/analysis/IcebergPartitionExpressionRewriter.java
index 83325e319..d1eb10d2e 100644
---
a/fe/src/main/java/org/apache/impala/analysis/IcebergPartitionExpressionRewriter.java
+++
b/fe/src/main/java/org/apache/impala/analysis/IcebergPartitionExpressionRewriter.java
@@ -87,8 +87,12 @@ class IcebergPartitionExpressionRewriter {
return rewrite(isNullPredicate);
}
if (expr instanceof InPredicate) {
- InPredicate isNullPredicate = (InPredicate) expr;
- return rewrite(isNullPredicate);
+ InPredicate inPredicate = (InPredicate) expr;
+ return rewrite(inPredicate);
+ }
+ if (expr instanceof LikePredicate) {
+ LikePredicate likePredicate = (LikePredicate) expr;
+ return rewrite(likePredicate);
}
throw new AnalysisException(
"Invalid partition filtering expression: " + expr.toSql());
@@ -160,6 +164,20 @@ class IcebergPartitionExpressionRewriter {
return inPredicate;
}
+ private LikePredicate rewrite(LikePredicate likePredicate)
+ throws AnalysisException {
+ Expr term = likePredicate.getChild(0);
+ IcebergPartitionExpr partitionExpr;
+ if (term instanceof SlotRef) {
+ partitionExpr = rewrite((SlotRef) term);
+ likePredicate.getChildren().set(0, partitionExpr);
+ } else if (term instanceof FunctionCallExpr) {
+ partitionExpr = rewrite((FunctionCallExpr) term);
+ likePredicate.getChildren().set(0, partitionExpr);
+ }
+ return likePredicate;
+ }
+
private void rewriteDateTransformConstants(LiteralExpr literal,
TIcebergPartitionTransformType transformType,
Function<NumericLiteral, ?> rewrite) {
diff --git a/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java
b/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java
index 0e37a17f1..41c812985 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java
@@ -147,12 +147,24 @@ public class ShowFilesStmt extends StatementBase
implements SingleTableStmt {
expr = rewriter.rewrite(expr);
expr.analyze(analyzer);
analyzer.getConstantFolder().rewrite(expr, analyzer);
- try {
- icebergPartitionExprs.add(converter.convert(expr));
- } catch (ImpalaException e) {
- throw new AnalysisException(
- "Invalid partition filtering expression: " + expr.toSql(), e);
+
+ IcebergPredicateConverter.ConverterResult result =
converter.convert(expr);
+
+ if (result.isFailed()) {
+ throw new AnalysisException(String.format(
+ "Invalid partition filtering expression: %s. %s",
+ expr.toSql(), result.getErrorMessage()));
}
+
+ if (result.isPartiallyConverted()) {
+ throw new AnalysisException(String.format(
+ "Predicate '%s' can only be partially converted to Iceberg
expression: " +
+ "'%s'. Partially converted predicates are not allowed in SHOW
FILES as " +
+ "they could show more files than intended. Use a fully convertible
" +
+ "predicate instead.", expr.toSql(),
result.getIcebergExpression()));
+ }
+
+ icebergPartitionExprs.add(result.getIcebergExpression());
}
try (CloseableIterable<FileScanTask> fileScanTasks =
IcebergUtil.planFiles(table,
diff --git a/fe/src/main/java/org/apache/impala/analysis/ShowStatsStmt.java
b/fe/src/main/java/org/apache/impala/analysis/ShowStatsStmt.java
index e84383ae6..f6f746333 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ShowStatsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ShowStatsStmt.java
@@ -34,6 +34,7 @@ import org.apache.impala.catalog.FeView;
import org.apache.impala.catalog.paimon.FePaimonTable;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.IcebergPartitionPredicateConverter;
+import org.apache.impala.common.IcebergPredicateConverter;
import org.apache.impala.planner.HdfsPartitionPruner;
import org.apache.impala.rewrite.ExprRewriter;
import org.apache.impala.rewrite.ExtractCompoundVerticalBarExprRule;
@@ -291,8 +292,23 @@ public class ShowStatsStmt extends StatementBase
implements SingleTableStmt {
// BoolLiterals are handled by the converter and optimized in
getPartitionStats
IcebergPartitionPredicateConverter converter =
new IcebergPartitionPredicateConverter(table.getIcebergSchema(),
analyzer);
+ IcebergPredicateConverter.ConverterResult result =
converter.convert(foldedExpr);
+
+ if (result.isFailed()) {
+ throw new AnalysisException(String.format(
+ "Invalid SHOW STATS expression: %s. %s",
+ foldedExpr.toSql(), result.getErrorMessage()));
+ }
+
+ if (result.isPartiallyConverted()) {
+ throw new AnalysisException(String.format(
+ "Cannot use LIKE predicate with literal content after wildcard in
SHOW " +
+ "STATS. Given predicate %s cannot be converted to an Iceberg
expression.",
+ foldedExpr.toSql()));
+ }
+
org.apache.iceberg.expressions.Expression icebergExpr =
- converter.convert(foldedExpr);
+ result.getIcebergExpression();
// Compute the filtered partition stats using the Iceberg Expression
filteredIcebergPartitionStats_ =
diff --git
a/fe/src/main/java/org/apache/impala/common/IcebergPredicateConverter.java
b/fe/src/main/java/org/apache/impala/common/IcebergPredicateConverter.java
index 946fcf41e..34b0a2d37 100644
--- a/fe/src/main/java/org/apache/impala/common/IcebergPredicateConverter.java
+++ b/fe/src/main/java/org/apache/impala/common/IcebergPredicateConverter.java
@@ -58,16 +58,81 @@ public class IcebergPredicateConverter {
private final Schema schema_;
private final Analyzer analyzer_;
+ /**
+ * Status of the conversion result.
+ * FULLY_CONVERTED: The predicate was fully converted to an Iceberg
expression that is
+ * semantically equivalent to the original predicate. No additional
evaluation needed
+ * by Impala.
+ * PARTIALLY_CONVERTED: The predicate was partially converted to a relaxed
Iceberg
+ * expression that is less restrictive than the original. The original
predicate must
+ * still be evaluated by Impala on surviving rows.
+ * FAILED: The predicate could not be converted to an Iceberg expression.
+ */
+ public enum ConversionStatus {
+ FULLY_CONVERTED,
+ PARTIALLY_CONVERTED,
+ FAILED
+ }
+
+ /**
+ * Result of converting an Impala predicate to an Iceberg expression.
+ */
+ public static class ConverterResult {
+ private final Expression icebergExpression;
+ private final ConversionStatus status;
+ private final String errorMessage;
+
+ public ConverterResult(Expression icebergExpression, ConversionStatus
status) {
+ this(icebergExpression, status, null);
+ }
+
+ public ConverterResult(ConversionStatus status, String errorMessage) {
+ this(null, status, errorMessage);
+ }
+
+ private ConverterResult(Expression icebergExpression, ConversionStatus
status,
+ String errorMessage) {
+ this.icebergExpression = icebergExpression;
+ this.status = status;
+ this.errorMessage = errorMessage;
+ }
+
+ public Expression getIcebergExpression() {
+ return icebergExpression;
+ }
+
+ public ConversionStatus getStatus() {
+ return status;
+ }
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+
+ public boolean isFullyConverted() {
+ return status == ConversionStatus.FULLY_CONVERTED;
+ }
+
+ public boolean isPartiallyConverted() {
+ return status == ConversionStatus.PARTIALLY_CONVERTED;
+ }
+
+ public boolean isFailed() {
+ return status == ConversionStatus.FAILED;
+ }
+ }
+
public IcebergPredicateConverter(Schema schema, Analyzer analyzer) {
this.schema_ = schema;
this.analyzer_ = analyzer;
}
- public Expression convert(Expr expr) throws ImpalaRuntimeException {
+ public ConverterResult convert(Expr expr) {
if (expr instanceof BoolLiteral) {
BoolLiteral boolLiteral = (BoolLiteral) expr;
- return boolLiteral.getValue() ? Expressions.alwaysTrue() :
+ Expression iceExpr = boolLiteral.getValue() ? Expressions.alwaysTrue() :
Expressions.alwaysFalse();
+ return new ConverterResult(iceExpr, ConversionStatus.FULLY_CONVERTED);
} else if (expr instanceof BinaryPredicate) {
return convert((BinaryPredicate) expr);
} else if (expr instanceof InPredicate) {
@@ -79,78 +144,119 @@ public class IcebergPredicateConverter {
} else if (expr instanceof LikePredicate) {
return convert((LikePredicate) expr);
} else {
- throw new ImpalaRuntimeException(String.format(
- "Unsupported expression: %s", expr.toSql()));
+ return new ConverterResult(ConversionStatus.FAILED,
+ String.format("Unsupported expression: %s", expr.toSql()));
}
}
- protected Expression convert(BinaryPredicate predicate) throws
ImpalaRuntimeException {
- Term term = getTerm(predicate.getChild(0));
- IcebergColumn column = term.referencedColumn_;
+ protected ConverterResult convert(BinaryPredicate predicate) {
+ try {
+ Term term = getTerm(predicate.getChild(0));
+ IcebergColumn column = term.referencedColumn_;
- LiteralExpr literal = getSecondChildAsLiteralExpr(predicate);
- checkNullLiteral(literal);
- Operation op = getOperation(predicate);
- Object value = getIcebergValue(column, literal);
+ LiteralExpr literal = getSecondChildAsLiteralExpr(predicate);
+ checkNullLiteral(literal);
+ Operation op = getOperation(predicate);
+ Object value = getIcebergValue(column, literal);
- List<Object> literals = Collections.singletonList(value);
- return Expressions.predicate(op, term.term_, literals);
+ List<Object> literals = Collections.singletonList(value);
+ Expression iceExpr = Expressions.predicate(op, term.term_, literals);
+ return new ConverterResult(iceExpr, ConversionStatus.FULLY_CONVERTED);
+ } catch (ImpalaRuntimeException e) {
+ return new ConverterResult(ConversionStatus.FAILED, e.getMessage());
+ }
}
- protected UnboundPredicate<Object> convert(InPredicate predicate)
- throws ImpalaRuntimeException {
- Term term = getTerm(predicate.getChild(0));
- IcebergColumn column = term.referencedColumn_;
- // Expressions takes a list of values as Objects
- List<Object> values = new ArrayList<>();
- for (int i = 1; i < predicate.getChildren().size(); ++i) {
- if (!Expr.IS_LITERAL.apply(predicate.getChild(i))) {
- throw new ImpalaRuntimeException(
- String.format("Expression is not a literal: %s",
- predicate.getChild(i)));
+ protected ConverterResult convert(InPredicate predicate) {
+ try {
+ Term term = getTerm(predicate.getChild(0));
+ IcebergColumn column = term.referencedColumn_;
+ // Expressions takes a list of values as Objects
+ List<Object> values = new ArrayList<>();
+ for (int i = 1; i < predicate.getChildren().size(); ++i) {
+ if (!Expr.IS_LITERAL.apply(predicate.getChild(i))) {
+ return new ConverterResult(ConversionStatus.FAILED,
+ String.format("Expression is not a literal: %s",
+ predicate.getChild(i)));
+ }
+ LiteralExpr literal = (LiteralExpr) predicate.getChild(i);
+ checkNullLiteral(literal);
+ Object value = getIcebergValue(column, literal);
+ values.add(value);
}
- LiteralExpr literal = (LiteralExpr) predicate.getChild(i);
- checkNullLiteral(literal);
- Object value = getIcebergValue(column, literal);
- values.add(value);
- }
- // According to the method:
- //
'org.apache.iceberg.expressions.InclusiveMetricsEvaluator.MetricsEvalVisitor#notIn'
- // Expressions.notIn only works when the push-down column is the partition
column
- if (predicate.isNotIn()) {
- return Expressions.notIn(term.term_, values);
- } else {
- return Expressions.in(term.term_, values);
+ // According to the method:
+ // 'org.apache.iceberg.expressions.InclusiveMetricsEvaluator
+ // .MetricsEvalVisitor#notIn'
+ // Expressions.notIn only works when the push-down column is the
partition column
+ UnboundPredicate<Object> iceExpr;
+ if (predicate.isNotIn()) {
+ iceExpr = Expressions.notIn(term.term_, values);
+ } else {
+ iceExpr = Expressions.in(term.term_, values);
+ }
+ return new ConverterResult(iceExpr, ConversionStatus.FULLY_CONVERTED);
+ } catch (ImpalaRuntimeException e) {
+ return new ConverterResult(ConversionStatus.FAILED, e.getMessage());
}
}
- protected UnboundPredicate<Object> convert(IsNullPredicate predicate)
- throws ImpalaRuntimeException {
- Term term = getTerm(predicate.getChild(0));
- if (predicate.isNotNull()) {
- return Expressions.notNull(term.term_);
- } else {
- return Expressions.isNull(term.term_);
+ protected ConverterResult convert(IsNullPredicate predicate) {
+ try {
+ Term term = getTerm(predicate.getChild(0));
+ UnboundPredicate<Object> iceExpr;
+ if (predicate.isNotNull()) {
+ iceExpr = Expressions.notNull(term.term_);
+ } else {
+ iceExpr = Expressions.isNull(term.term_);
+ }
+ return new ConverterResult(iceExpr, ConversionStatus.FULLY_CONVERTED);
+ } catch (ImpalaRuntimeException e) {
+ return new ConverterResult(ConversionStatus.FAILED, e.getMessage());
}
}
- protected Expression convert(CompoundPredicate predicate)
- throws ImpalaRuntimeException {
- Operation op = getOperation(predicate);
+ protected ConverterResult convert(CompoundPredicate predicate) {
+ try {
+ Operation op = getOperation(predicate);
- Expr leftExpr = predicate.getChild(0);
- Expression left = convert(leftExpr);
+ Expr leftExpr = predicate.getChild(0);
+ ConverterResult leftResult = convert(leftExpr);
- if (op.equals(Operation.NOT)) {
- return Expressions.not(left);
- }
+ // If left child failed, propagate failure
+ if (leftResult.isFailed()) {
+ return leftResult;
+ }
- Expr rightExpr = predicate.getChild(1);
- Expression right = convert(rightExpr);
+ if (op.equals(Operation.NOT)) {
+ Expression iceExpr =
Expressions.not(leftResult.getIcebergExpression());
+ return new ConverterResult(iceExpr, leftResult.getStatus());
+ }
- return op.equals(Operation.AND) ? Expressions.and(left, right) :
- Expressions.or(left, right);
+ Expr rightExpr = predicate.getChild(1);
+ ConverterResult rightResult = convert(rightExpr);
+
+ // If right child failed, propagate failure
+ if (rightResult.isFailed()) {
+ return rightResult;
+ }
+
+ Expression iceExpr = op.equals(Operation.AND) ?
+ Expressions.and(leftResult.getIcebergExpression(),
+ rightResult.getIcebergExpression()) :
+ Expressions.or(leftResult.getIcebergExpression(),
+ rightResult.getIcebergExpression());
+
+ // If either child is partially converted, the compound predicate
+ // is partially converted
+ ConversionStatus status = ConversionStatus.FULLY_CONVERTED;
+ if (leftResult.isPartiallyConverted() ||
rightResult.isPartiallyConverted()) {
+ status = ConversionStatus.PARTIALLY_CONVERTED;
+ }
+ return new ConverterResult(iceExpr, status);
+ } catch (ImpalaRuntimeException e) {
+ return new ConverterResult(ConversionStatus.FAILED, e.getMessage());
+ }
}
/**
@@ -258,64 +364,71 @@ public class IcebergPredicateConverter {
return sb.toString();
}
- protected Expression convert(LikePredicate predicate)
- throws ImpalaRuntimeException {
- // Only LIKE operator is supported, not RLIKE, REGEXP, etc.
- if (predicate.getOp() != LikePredicate.Operator.LIKE) {
- throw new ImpalaRuntimeException(String.format(
- "Only LIKE operator is supported for Iceberg pushdown, got: %s",
- predicate.getOp()));
- }
+ protected ConverterResult convert(LikePredicate predicate) {
+ try {
+ // Only LIKE operator is supported, not RLIKE, REGEXP, etc.
+ if (predicate.getOp() != LikePredicate.Operator.LIKE) {
+ return new ConverterResult(ConversionStatus.FAILED,
+ String.format("Only LIKE operator is supported for Iceberg
pushdown, got: %s",
+ predicate.getOp()));
+ }
- Term term = getTerm(predicate.getChild(0));
- IcebergColumn column = term.referencedColumn_;
+ Term term = getTerm(predicate.getChild(0));
+ IcebergColumn column = term.referencedColumn_;
- // Check if the column is a string type
- if (!column.getType().isStringType()) {
- throw new ImpalaRuntimeException(String.format(
- "LIKE predicate pushdown only supports string columns, got: %s",
- column.getType()));
- }
+ // Check if the column is a string type
+ if (!column.getType().isStringType()) {
+ return new ConverterResult(ConversionStatus.FAILED,
+ String.format("LIKE predicate pushdown only supports string
columns, got: %s",
+ column.getType()));
+ }
- LiteralExpr literal = getSecondChildAsLiteralExpr(predicate);
- checkNullLiteral(literal);
+ LiteralExpr literal = getSecondChildAsLiteralExpr(predicate);
+ checkNullLiteral(literal);
- if (!(literal instanceof StringLiteral)) {
- throw new ImpalaRuntimeException(String.format(
- "LIKE pattern must be a string literal, got: %s", literal.toSql()));
- }
+ if (!(literal instanceof StringLiteral)) {
+ return new ConverterResult(ConversionStatus.FAILED,
+ String.format("LIKE pattern must be a string literal, got: %s",
+ literal.toSql()));
+ }
- String pattern = ((StringLiteral) literal).getUnescapedValue();
- if (pattern == null || pattern.isEmpty()) {
- throw new ImpalaRuntimeException("LIKE pattern cannot be null or empty");
- }
+ String pattern = ((StringLiteral) literal).getUnescapedValue();
+ if (pattern == null || pattern.isEmpty()) {
+ return new ConverterResult(ConversionStatus.FAILED,
+ "LIKE pattern cannot be null or empty");
+ }
- // Find first unescaped wildcard position
- int firstWildcard = findFirstUnescapedWildcard(pattern);
+ // Find first unescaped wildcard position
+ int firstWildcard = findFirstUnescapedWildcard(pattern);
- // Case 1: Pattern starts with wildcard - cannot push down
- if (firstWildcard == 0) {
- throw new ImpalaRuntimeException(String.format(
- "LIKE pattern '%s' cannot be pushed down to Iceberg. Patterns must
start "
- + "with at least one literal character.", pattern));
- }
+ // Case 1: Pattern starts with wildcard - cannot push down
+ if (firstWildcard == 0) {
+ return new ConverterResult(ConversionStatus.FAILED,
+ String.format("LIKE pattern '%s' cannot be pushed down to Iceberg.
" +
+ "Patterns must start with at least one literal character.",
pattern));
+ }
- // Case 2: No wildcards - exact match
- if (firstWildcard == -1) {
- String unescapedPattern = unescapeLikePattern(pattern, -1);
- return Expressions.equal(column.getName(), unescapedPattern);
- }
+ // Case 2: No wildcards - exact match
+ if (firstWildcard == -1) {
+ String unescapedPattern = unescapeLikePattern(pattern, -1);
+ Expression iceExpr = Expressions.equal(column.getName(),
unescapedPattern);
+ return new ConverterResult(iceExpr, ConversionStatus.FULLY_CONVERTED);
+ }
- // Case 3: Wildcard in middle with literal content after - cannot push down
- if (hasLiteralContentAfterWildcard(pattern, firstWildcard)) {
- throw new ImpalaRuntimeException(String.format(
- "LIKE pattern '%s' cannot be pushed down to Iceberg. Only prefix
patterns "
- + "(e.g., 'prefix%%') are supported.", pattern));
+ // Case 3: Wildcard in middle with literal content after - push down
relaxed prefix
+ // This allows Iceberg to prune partitions/files, but the original LIKE
predicate
+ // will still be evaluated by Impala on the surviving rows.
+ // Example: LIKE 'd%d' pushes down startsWith('d'), keeps LIKE 'd%d' in
scan
+ boolean hasContentAfterWildcard = hasLiteralContentAfterWildcard(pattern,
+ firstWildcard);
+ ConversionStatus status = hasContentAfterWildcard ?
+ ConversionStatus.PARTIALLY_CONVERTED :
ConversionStatus.FULLY_CONVERTED;
+ String unescapedPattern = unescapeLikePattern(pattern, firstWildcard);
+ Expression iceExpr = Expressions.startsWith(column.getName(),
unescapedPattern);
+ return new ConverterResult(iceExpr, status);
+ } catch (ImpalaRuntimeException e) {
+ return new ConverterResult(ConversionStatus.FAILED, e.getMessage());
}
-
- // Case 4: Pure prefix pattern (wildcard only at end) - use startsWith
- String unescapedPattern = unescapeLikePattern(pattern, firstWildcard);
- return Expressions.startsWith(column.getName(), unescapedPattern);
}
protected void checkNullLiteral(LiteralExpr literal) throws
ImpalaRuntimeException {
diff --git a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
index d790250d4..27ec98de4 100644
--- a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
@@ -143,6 +143,10 @@ public class IcebergScanPlanner {
private final List<Expr> skippedExpressions_ = new ArrayList<>();
// Impala expressions that can't be translated into Iceberg expressions
private final List<Expr> untranslatedExpressions_ = new ArrayList<>();
+ // Impala expressions that were relaxed when pushed to Iceberg
+ // (e.g., LIKE 'd%d' -> startsWith('d'))
+ // These must be evaluated by Impala on the surviving rows
+ private final List<Expr> relaxedExpressions_ = new ArrayList<>();
// Conjuncts on columns not involved in IDENTITY-partitioning.
private List<Expr> nonIdentityConjuncts_ = new ArrayList<>();
@@ -790,7 +794,11 @@ public class IcebergScanPlanner {
private void filterConjuncts() {
if (residualExpressions_.isEmpty()) {
- conjuncts_.removeAll(impalaIcebergPredicateMapping_.values());
+ // Remove fully pushed predicates but keep relaxed ones for evaluation
+ List<Expr> toRemove = impalaIcebergPredicateMapping_.values().stream()
+ .filter(expr -> !relaxedExpressions_.contains(expr))
+ .collect(Collectors.toList());
+ conjuncts_.removeAll(toRemove);
return;
}
if (!analyzer_.getQueryOptions().iceberg_predicate_pushdown_subsetting)
return;
@@ -800,6 +808,8 @@ public class IcebergScanPlanner {
private boolean trySubsettingPredicatesBeingPushedDown() {
long startTime = System.currentTimeMillis();
List<Expr> expressionsToRetain = new ArrayList<>(untranslatedExpressions_);
+ // Add relaxed predicates - they were pushed to Iceberg but must still be
evaluated
+ expressionsToRetain.addAll(relaxedExpressions_);
for (Expression expression : residualExpressions_) {
List<Expression> locatedExpressions =
ExpressionVisitors.visit(expression,
new IcebergExpressionCollector());
@@ -822,7 +832,10 @@ public class IcebergScanPlanner {
private List<Expr> getSkippedConjuncts() {
if (!residualExpressions_.isEmpty()) return skippedExpressions_;
- return new ArrayList<>(impalaIcebergPredicateMapping_.values());
+ // Return all pushed predicates except relaxed ones (which still need
evaluation)
+ return impalaIcebergPredicateMapping_.values().stream()
+ .filter(expr -> !relaxedExpressions_.contains(expr))
+ .collect(Collectors.toList());
}
private void updateDeleteStatistics() {
@@ -1088,21 +1101,33 @@ public class IcebergScanPlanner {
}
/**
- * Transform Impala predicate to Iceberg predicate
+ * Transform Impala predicate to Iceberg predicate.
+ * Returns true if a predicate was pushed to Iceberg (even if partially
converted).
+ * If the predicate was partially converted (e.g., LIKE 'd%d' ->
startsWith('d')),
+ * the original predicate is kept in relaxedExpressions_ so it will
+ * be evaluated by Impala on the surviving rows after Iceberg pruning.
*/
private boolean tryConvertIcebergPredicate(Expr expr) {
IcebergPredicateConverter converter =
new IcebergPredicateConverter(getIceTable().getIcebergSchema(),
analyzer_);
- try {
- Expression predicate = converter.convert(expr);
- impalaIcebergPredicateMapping_.put(predicate, expr);
- LOG.debug("Push down the predicate: {} to iceberg", predicate);
- return true;
- }
- catch (ImpalaException e) {
+ IcebergPredicateConverter.ConverterResult result = converter.convert(expr);
+
+ if (result.isFailed()) {
untranslatedExpressions_.add(expr);
return false;
}
+
+ Expression icebergExpr = result.getIcebergExpression();
+ impalaIcebergPredicateMapping_.put(icebergExpr, expr);
+ LOG.debug("Push down the predicate: {} to iceberg (status={})",
+ icebergExpr, result.getStatus());
+
+ // If the predicate was partially converted, we must keep the original
predicate
+ // for evaluation by Impala on the pruned data
+ if (result.isPartiallyConverted()) {
+ relaxedExpressions_.add(expr);
+ }
+ return true;
}
}
diff --git
a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates.test
b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates.test
index 1180e6690..3024f8356 100644
---
a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates.test
+++
b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-predicates.test
@@ -141,13 +141,14 @@ PLAN-ROOT SINK
Iceberg snapshot id: 8270633197658268308
row-size=44B cardinality=1
====
-# LIKE with wildcard in middle (d%d) cannot be pushed down - stays in
predicates
+# LIKE with wildcard in middle (d%d) pushes down relaxed prefix for pruning
+# Relaxed to startsWith('d') for Iceberg, but full LIKE 'd%d' evaluated by
Impala
select * from iceberg_partitioned where action like "d%d" and event_time <
"2022-01-01" and id < 10
---- PLAN
PLAN-ROOT SINK
|
00:SCAN HDFS [functional_parquet.iceberg_partitioned]
- HDFS partitions=3/3 files=9 size=10.33KB
+ HDFS partitions=1/3 files=4 size=4.65KB
predicates: id < 10, action LIKE 'd%d'
Iceberg snapshot id: 8270633197658268308
skipped Iceberg predicates: event_time < TIMESTAMP '2022-01-01 00:00:00'
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-drop-partition.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-drop-partition.test
index 44f0020ab..71f99d5bb 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-drop-partition.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-drop-partition.test
@@ -496,3 +496,63 @@ ALTER TABLE
$DATABASE.iceberg_drop_partition_between_bucket_test DROP PARTITION
---- CATCH
AnalysisException: $DATABASE.bucket() unknown for database $DATABASE.
Currently this db has 0 functions.
====
+---- QUERY
+# Test LIKE predicate with simple prefix pattern (should work)
+CREATE TABLE $DATABASE.iceberg_drop_partition_like_test (s STRING)
+PARTITIONED BY SPEC (s)
+STORED AS ICEBERG;
+====
+---- QUERY
+INSERT INTO $DATABASE.iceberg_drop_partition_like_test VALUES ('download'),
('delete'), ('data123'), ('upload');
+====
+---- QUERY
+# Simple prefix patterns should work fine - drops partitions starting with 'd'
+ALTER TABLE $DATABASE.iceberg_drop_partition_like_test DROP PARTITION (s LIKE
'd%');
+---- RESULTS
+'Dropped 3 partition(s)'
+====
+---- QUERY
+# Verify only partitions starting with 'd' were dropped
+SELECT s FROM $DATABASE.iceberg_drop_partition_like_test ORDER BY s;
+---- RESULTS
+'upload'
+====
+---- QUERY
+# Negative test: Literal values are not allowed after wildcard in LIKE pattern
for DROP PARTITION
+ALTER TABLE $DATABASE.iceberg_drop_partition_like_test DROP PARTITION (s LIKE
'd%d');
+---- CATCH
+AnalysisException: Predicate 's LIKE 'd%d'' can only be partially converted to
Iceberg expression
+====
+---- QUERY
+# Negative test: pattern 'test%value' also has literal after the wildcard
+ALTER TABLE $DATABASE.iceberg_drop_partition_like_test DROP PARTITION (s LIKE
'test%value');
+---- CATCH
+AnalysisException: Predicate 's LIKE 'test%value'' can only be partially
converted to Iceberg expression
+====
+---- QUERY
+# Test LIKE predicate with TRUNCATE partition spec
+CREATE TABLE $DATABASE.iceberg_drop_partition_like_truncate (s STRING)
+PARTITIONED BY SPEC (TRUNCATE(1, s))
+STORED AS ICEBERG;
+====
+---- QUERY
+INSERT INTO $DATABASE.iceberg_drop_partition_like_truncate VALUES
('download'), ('delete'), ('data123'), ('upload'), ('document');
+====
+---- QUERY
+# All values starting with 'd' truncate to 'd' and share the same partition
+ALTER TABLE $DATABASE.iceberg_drop_partition_like_truncate DROP PARTITION (s
LIKE 'd%');
+---- RESULTS
+'Dropped 1 partition(s)'
+====
+---- QUERY
+# Verify correct partition was dropped
+SELECT s FROM $DATABASE.iceberg_drop_partition_like_truncate ORDER BY s;
+---- RESULTS
+'upload'
+====
+---- QUERY
+# Negative test
+ALTER TABLE $DATABASE.iceberg_drop_partition_like_truncate DROP PARTITION (s
LIKE 'd%d');
+---- CATCH
+AnalysisException: Predicate 's LIKE 'd%d'' can only be partially converted to
Iceberg expression
+====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-like-pushdown.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-like-pushdown.test
index 01454ec48..a8747f4e9 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-like-pushdown.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-like-pushdown.test
@@ -327,23 +327,32 @@ INT, STRING
20,'Alex'
====
---- QUERY
+# Test: LIKE with literal after wildcard (árvíz%p) - pushes down relaxed prefix
+# Pattern is relaxed to startsWith('árvíz') for Iceberg pruning,
+# but full LIKE 'árvíz%p' is still evaluated by Impala to ensure 'p' at end
# Verify árvíz%p returns correct results (should only match
'árvíztűrőtükörfúrógép' and not 'árvíztűrő')
-# Checks pattern not being pushed down, else startsWith('árvíz') would
incorrectly match 'árvíztűrő'
SELECT s FROM ice_utf8_test WHERE s LIKE 'árvíz%p';
---- TYPES
STRING
---- RESULTS: RAW_STRING
'árvíztűrőtükörfúrógép'
+---- RUNTIME_PROFILE
+row_regex: .*partitions=1.*files=.*
+row_regex: .*predicates:.*s LIKE.*
====
---- QUERY
-# Negative Test: LIKE with suffix after wildcard - cannot be pushed down
-# Using escaped wildcard test table to verify patterns with literal content
after wildcard
+# Test: LIKE with suffix after wildcard - pushes down relaxed prefix
+# Pattern 'test%value' is relaxed to startsWith('test') for Iceberg pruning
+# Verify test%value returns correct results
SELECT s FROM ice_utf8_test WHERE s LIKE 'test%value' ORDER BY s;
---- TYPES
STRING
---- RESULTS
'test%value'
'test_value'
+---- RUNTIME_PROFILE
+row_regex: .*partitions=2.*files=.*
+row_regex: .*predicates:.*s LIKE.*
====
---- QUERY
# Negative Test: Complex pattern with wildcards and literal content
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-show-files-partition.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-show-files-partition.test
index 4cba6a15a..c3dcc072f 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-show-files-partition.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-show-files-partition.test
@@ -470,4 +470,55 @@
row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/iceberg_showfiles_with_deletes/
row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/iceberg_showfiles_with_deletes/data/identity_int=1/(?!delete).*_data.*.parq','.*','','$ERASURECODE_POLICY'
---- TYPES
STRING, STRING, STRING, STRING
-====
\ No newline at end of file
+====
+---- QUERY
+# Test LIKE predicate with simple prefix pattern (should work)
+CREATE TABLE $DATABASE.iceberg_showfiles_like_test (s STRING)
+PARTITIONED BY SPEC (s)
+STORED AS ICEBERG;
+====
+---- QUERY
+INSERT INTO $DATABASE.iceberg_showfiles_like_test VALUES ('download'),
('delete'), ('data123'), ('upload');
+====
+---- QUERY
+# Simple prefix patterns should work - shows files from partitions starting
with 'd'
+SHOW FILES IN $DATABASE.iceberg_showfiles_like_test PARTITION (s LIKE 'd%');
+---- TYPES
+STRING, STRING, STRING, STRING
+---- RESULTS
+row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/iceberg_showfiles_like_test/data/.*s=data123.*_data.*.parq','.*','','$ERASURECODE_POLICY'
+row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/iceberg_showfiles_like_test/data/.*s=delete.*_data.*.parq','.*','','$ERASURECODE_POLICY'
+row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/iceberg_showfiles_like_test/data/.*s=download.*_data.*.parq','.*','','$ERASURECODE_POLICY'
+====
+---- QUERY
+# Negative test: Literal values are not allowed after wildcard in LIKE pattern
for SHOW FILES
+SHOW FILES IN $DATABASE.iceberg_showfiles_like_test PARTITION (s LIKE 'd%d');
+---- CATCH
+AnalysisException: Predicate 's LIKE 'd%d'' can only be partially converted to
Iceberg expression
+====
+---- QUERY
+# Test LIKE predicate with TRUNCATE partition spec
+CREATE TABLE $DATABASE.iceberg_showfiles_like_truncate (s STRING)
+PARTITIONED BY SPEC (TRUNCATE(3, s))
+STORED AS ICEBERG;
+====
+---- QUERY
+INSERT INTO $DATABASE.iceberg_showfiles_like_truncate VALUES ('download'),
('delete'), ('data123'), ('upload'), ('document');
+====
+---- QUERY
+# Shows files from partitions where truncate(3,s) matches
+SHOW FILES IN $DATABASE.iceberg_showfiles_like_truncate PARTITION (s LIKE
'd%');
+---- TYPES
+STRING, STRING, STRING, STRING
+---- RESULTS
+row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/iceberg_showfiles_like_truncate/data/.*s_trunc=dat.*_data.*.parq','.*','','$ERASURECODE_POLICY'
+row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/iceberg_showfiles_like_truncate/data/.*s_trunc=del.*_data.*.parq','.*','','$ERASURECODE_POLICY'
+row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/iceberg_showfiles_like_truncate/data/.*s_trunc=doc.*_data.*.parq','.*','','$ERASURECODE_POLICY'
+row_regex:'$NAMENODE/test-warehouse/$DATABASE.db/iceberg_showfiles_like_truncate/data/.*s_trunc=dow.*_data.*.parq','.*','','$ERASURECODE_POLICY'
+====
+---- QUERY
+# Negative test
+SHOW FILES IN $DATABASE.iceberg_showfiles_like_truncate PARTITION (s LIKE
'd%d');
+---- CATCH
+AnalysisException: Predicate 's LIKE 'd%d'' can only be partially converted to
Iceberg expression
+====
