This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3ca9113a94 [VL] Delta: Fix DeleteSQLNameColumnMappingSuite (#10822)
3ca9113a94 is described below
commit 3ca9113a946ce1d402866a52883af70788a2eee0
Author: Hongze Zhang <[email protected]>
AuthorDate: Wed Oct 1 11:26:22 2025 +0200
[VL] Delta: Fix DeleteSQLNameColumnMappingSuite (#10822)
---
.../org/apache/spark/sql/delta/DeleteSQLSuite.scala | 5 -----
.../gluten/extension/DeltaPostTransformRules.scala | 20 ++++++++++++++++----
2 files changed, 16 insertions(+), 9 deletions(-)
diff --git
a/backends-velox/src-delta33/test/scala/org/apache/spark/sql/delta/DeleteSQLSuite.scala
b/backends-velox/src-delta33/test/scala/org/apache/spark/sql/delta/DeleteSQLSuite.scala
index 950c2269d7..7bd6a59b7b 100644
---
a/backends-velox/src-delta33/test/scala/org/apache/spark/sql/delta/DeleteSQLSuite.scala
+++
b/backends-velox/src-delta33/test/scala/org/apache/spark/sql/delta/DeleteSQLSuite.scala
@@ -20,8 +20,6 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.delta.sources.DeltaSQLConf
import org.apache.spark.sql.delta.test.{DeltaExcludedTestMixin,
DeltaSQLCommandTest}
-import org.scalatest.Ignore
-
// spotless:off
class DeleteSQLSuite extends DeleteSuiteBase
with DeltaExcludedTestMixin
@@ -95,9 +93,6 @@ class DeleteSQLSuite extends DeleteSuiteBase
}
}
-// FIXME: Enable the test.
-// Skipping as function input_file_name doesn't get correctly resolved.
-@Ignore
class DeleteSQLNameColumnMappingSuite extends DeleteSQLSuite
with DeltaColumnMappingEnableNameMode {
diff --git
a/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala
b/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala
index 83af341a78..156fdf791a 100644
---
a/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala
+++
b/gluten-delta/src/main/scala/org/apache/gluten/extension/DeltaPostTransformRules.scala
@@ -31,7 +31,7 @@ import scala.collection.mutable.ListBuffer
object DeltaPostTransformRules {
def rules: Seq[Rule[SparkPlan]] =
- RemoveTransitions :: columnMappingRule :: pushDownInputFileExprRule :: Nil
+ RemoveTransitions :: pushDownInputFileExprRule :: columnMappingRule :: Nil
private val COLUMN_MAPPING_RULE_TAG: TreeNodeTag[String] =
TreeNodeTag[String]("org.apache.gluten.delta.column.mapping")
@@ -76,6 +76,16 @@ object DeltaPostTransformRules {
}
}
+ private def isInputFileRelatedAttribute(attr: Attribute): Boolean = {
+ attr match {
+ case AttributeReference(name, _, _, _) =>
+ Seq(InputFileName(), InputFileBlockStart(), InputFileBlockLength())
+ .map(_.prettyName)
+ .contains(name)
+ case _ => false
+ }
+ }
+
private[gluten] def containsIncrementMetricExpr(expr: Expression): Boolean =
{
expr match {
case e if e.prettyName == "increment_metric" => true
@@ -108,12 +118,14 @@ object DeltaPostTransformRules {
val originColumnNames = ListBuffer.empty[String]
val transformedAttrs = ListBuffer.empty[Attribute]
def mapAttribute(attr: Attribute) = {
- val newAttr = if (!plan.isMetadataColumn(attr)) {
+ val newAttr = if (plan.isMetadataColumn(attr)) {
+ attr
+ } else if (isInputFileRelatedAttribute(attr)) {
+ attr
+ } else {
DeltaColumnMapping
.createPhysicalAttributes(Seq(attr), fmt.referenceSchema,
fmt.columnMappingMode)
.head
- } else {
- attr
}
if (!originColumnNames.contains(attr.name)) {
transformedAttrs += newAttr
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
