This is an automated email from the ASF dual-hosted git repository.
slfan1989 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git
The following commit(s) were added to refs/heads/master by this push:
new fe9a2ff9 [AURON #2425] Support Iceberg STARTS_WITH pruning in native
scan (#2426)
fe9a2ff9 is described below
commit fe9a2ff9a15bf951427a9dcdeb8bb55644ce476e
Author: Ming Wei <[email protected]>
AuthorDate: Sun Jul 26 13:06:28 2026 +0800
[AURON #2425] Support Iceberg STARTS_WITH pruning in native scan (#2426)
**Which issue does this PR close?**
Closes #2425
**Rationale for this change**
Auron Iceberg native scan already collects Iceberg residual filters and
converts supported filters into native scan pruning predicates.
Iceberg can produce `STARTS_WITH` filters for prefix string predicates,
such as `LIKE 'a%'`, but Auron currently does not convert this Iceberg
operation. As a result, the query can still use a native post-scan
filter, but the starts-with predicate is not used for native scan
pruning.
**What changes are included in this PR?**
Adds Iceberg `STARTS_WITH` conversion for string columns in
`IcebergScanSupport`.
Reuses the existing Spark `StartsWith` native pruning conversion.
Keeps other string and binary pruning behavior unchanged.
Adds an Iceberg integration test for prefix LIKE pruning.
**Are there any user-facing changes?**
No user-facing API changes. More Iceberg prefix string filters can now
be passed into native scan pruning.
**How was this patch tested?**
UT.
Co-authored-by: Shilun Fan <[email protected]>
Signed-off-by: weimingdiit <[email protected]>
---
.../sql/auron/iceberg/IcebergScanSupport.scala | 11 +++++++++-
.../iceberg/AuronIcebergIntegrationSuite.scala | 25 ++++++++++++++++++++++
2 files changed, 35 insertions(+), 1 deletion(-)
diff --git
a/thirdparty/auron-iceberg/src/main/scala/org/apache/spark/sql/auron/iceberg/IcebergScanSupport.scala
b/thirdparty/auron-iceberg/src/main/scala/org/apache/spark/sql/auron/iceberg/IcebergScanSupport.scala
index e280ab1e..e087e161 100644
---
a/thirdparty/auron-iceberg/src/main/scala/org/apache/spark/sql/auron/iceberg/IcebergScanSupport.scala
+++
b/thirdparty/auron-iceberg/src/main/scala/org/apache/spark/sql/auron/iceberg/IcebergScanSupport.scala
@@ -27,7 +27,7 @@ import org.apache.iceberg.expressions.{And => IcebergAnd,
BoundPredicate, Expres
import org.apache.iceberg.spark.source.AuronIcebergSourceUtil
import org.apache.spark.internal.Logging
import org.apache.spark.sql.auron.{NativeConverters, Shims}
-import org.apache.spark.sql.catalyst.expressions.{And => SparkAnd,
AttributeReference, EqualTo, Expression => SparkExpression, GreaterThan,
GreaterThanOrEqual, In, IsNaN, IsNotNull, IsNull, LessThan, LessThanOrEqual,
Literal, Not => SparkNot, Or => SparkOr}
+import org.apache.spark.sql.catalyst.expressions.{And => SparkAnd,
AttributeReference, EqualTo, Expression => SparkExpression, GreaterThan,
GreaterThanOrEqual, In, IsNaN, IsNotNull, IsNull, LessThan, LessThanOrEqual,
Literal, Not => SparkNot, Or => SparkOr, StartsWith}
import org.apache.spark.sql.catalyst.trees.TreeNodeTag
import org.apache.spark.sql.connector.read.{InputPartition, Scan}
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
@@ -769,6 +769,15 @@ object IcebergScanSupport extends Logging {
dataType: DataType,
op: org.apache.iceberg.expressions.Expression.Operation,
literalValue: Any): Option[SparkExpression] = {
+ if (op == org.apache.iceberg.expressions.Expression.Operation.STARTS_WITH)
{
+ if (dataType == StringType) {
+ return toLiteral(literalValue, StringType)
+ .filter(_.value != null)
+ .map(StartsWith(attr, _))
+ }
+ return None
+ }
+
if (!supportsScanPruningLiteralType(dataType)) {
return None
}
diff --git
a/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/AuronIcebergIntegrationSuite.scala
b/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/AuronIcebergIntegrationSuite.scala
index 1140b67b..0032e289 100644
---
a/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/AuronIcebergIntegrationSuite.scala
+++
b/thirdparty/auron-iceberg/src/test/scala/org/apache/auron/iceberg/AuronIcebergIntegrationSuite.scala
@@ -466,6 +466,31 @@ class AuronIcebergIntegrationSuite
}
}
+ test("iceberg scan pushes STARTS_WITH filters into native scan pruning
predicates") {
+ withTable("local.db.t_residual_starts_with") {
+ sql("create table local.db.t_residual_starts_with (id int, v string)
using iceberg")
+ sql("""
+ |insert into local.db.t_residual_starts_with
+ |values (1, 'alpha'), (2, 'beta'), (3, 'atom'), (4, null)
+ |""".stripMargin)
+ val df = sql("""
+ |select * from local.db.t_residual_starts_with
+ |where v like 'a%'
+ |""".stripMargin)
+ checkAnswer(df, Seq(Row(1, "alpha"), Row(3, "atom")))
+ val nativeScanPlan = icebergScanPlan(df)
+ assert(nativeScanPlan.nonEmpty)
+ val pruningPredicateText =
nativeScanPlan.get.pruningPredicates.mkString("\n")
+ assert(
+ pruningPredicateText.contains("name: \"starts_with\"") &&
+ pruningPredicateText.contains("fun: StartsWith"),
+ pruningPredicateText)
+ val plan = df.queryExecution.executedPlan.toString()
+ assert(plan.contains("NativeIcebergTableScan"))
+ assert(plan.contains("NativeFilter"))
+ }
+ }
+
test("iceberg scan keeps native post-scan filter when only part of the
predicate is pushed") {
withTable("local.db.t_residual_partial_pushdown") {
sql("create table local.db.t_residual_partial_pushdown (id int, v
string) using iceberg")
