This is an automated email from the ASF dual-hosted git repository.

yma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new fc748bbb8a [GLUTEN-8352][ICEBERG] Fix read error when partition column 
was drop (#8353)
fc748bbb8a is described below

commit fc748bbb8a4f99a6788a440d8bcc9efe7d06f129
Author: liangyongyuan <[email protected]>
AuthorDate: Fri Dec 27 14:15:36 2024 +0800

    [GLUTEN-8352][ICEBERG] Fix read error when partition column was drop (#8353)
    
    * [GLUTEN-8352][ICEBERG] Fix read error when partition column was drop
    
    * add ut
    
    * fix
    
    * fix
---
 .../spark/source/GlutenIcebergSourceUtil.scala     | 12 ++++++++++
 .../org/apache/gluten/execution/IcebergSuite.scala | 26 ++++++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git 
a/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
 
b/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
index 0898e068b7..a53c464b65 100644
--- 
a/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
+++ 
b/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
@@ -111,12 +111,24 @@ object GlutenIcebergSourceUtil {
             // Iceberg will generate some non-table fields as partition 
fields, such as x_bucket,
             // which will not appear in readFields, they also cannot be 
filtered.
             val tableFields = 
spec.schema().columns().asScala.map(_.name()).toSet
+            val voidTransformFields = scan
+              .table()
+              .spec()
+              .fields()
+              .asScala
+              .filter(
+                f => {
+                  f.transform().isVoid
+                })
+              .map(_.name())
+              .toSet
             val partitionFields =
               spec
                 .partitionType()
                 .fields()
                 .asScala
                 .filter(f => !tableFields.contains(f.name) || 
readFields.contains(f.name()))
+                .filter(f => !voidTransformFields.contains(f.name()))
             partitionFields.foreach {
               field => 
TypeUtil.validatePartitionColumnType(field.`type`().typeId())
             }
diff --git 
a/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala
 
b/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala
index cb630a1475..170be9746b 100644
--- 
a/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala
+++ 
b/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala
@@ -489,4 +489,30 @@ abstract class IcebergSuite extends 
WholeStageTransformerSuite {
         }
     }
   }
+
+  test("test read v1 iceberg with partition drop") {
+    val testTable = "test_table_with_partition"
+    withTable(testTable) {
+      spark.sql(s"""
+                   |CREATE TABLE $testTable (id INT, data STRING, p1 STRING, 
p2 STRING)
+                   |USING iceberg
+                   |tblproperties (
+                   |  'format-version' = '1'
+                   |)
+                   |PARTITIONED BY (p1, p2);
+                   |""".stripMargin)
+      spark.sql(s"""
+                   |INSERT INTO $testTable VALUES
+                   |(1, 'test_data', 'test_p1', 'test_p2');
+                   |""".stripMargin)
+      spark.sql(s"""
+                   |ALTER TABLE $testTable DROP PARTITION FIELD p2
+                   |""".stripMargin)
+      val resultDf = spark.sql(s"SELECT id, data, p1, p2 FROM $testTable")
+      val result = resultDf.collect()
+
+      assert(result.length == 1)
+      assert(result.head.getString(3) == "test_p2")
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to