This is an automated email from the ASF dual-hosted git repository.
yma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new fc748bbb8a [GLUTEN-8352][ICEBERG] Fix read error when partition column
was drop (#8353)
fc748bbb8a is described below
commit fc748bbb8a4f99a6788a440d8bcc9efe7d06f129
Author: liangyongyuan <[email protected]>
AuthorDate: Fri Dec 27 14:15:36 2024 +0800
[GLUTEN-8352][ICEBERG] Fix read error when partition column was drop (#8353)
* [GLUTEN-8352][ICEBERG] Fix read error when partition column was drop
* add ut
* fix
* fix
---
.../spark/source/GlutenIcebergSourceUtil.scala | 12 ++++++++++
.../org/apache/gluten/execution/IcebergSuite.scala | 26 ++++++++++++++++++++++
2 files changed, 38 insertions(+)
diff --git
a/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
b/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
index 0898e068b7..a53c464b65 100644
---
a/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
+++
b/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala
@@ -111,12 +111,24 @@ object GlutenIcebergSourceUtil {
// Iceberg will generate some non-table fields as partition
fields, such as x_bucket,
// which will not appear in readFields, they also cannot be
filtered.
val tableFields =
spec.schema().columns().asScala.map(_.name()).toSet
+ val voidTransformFields = scan
+ .table()
+ .spec()
+ .fields()
+ .asScala
+ .filter(
+ f => {
+ f.transform().isVoid
+ })
+ .map(_.name())
+ .toSet
val partitionFields =
spec
.partitionType()
.fields()
.asScala
.filter(f => !tableFields.contains(f.name) ||
readFields.contains(f.name()))
+ .filter(f => !voidTransformFields.contains(f.name()))
partitionFields.foreach {
field =>
TypeUtil.validatePartitionColumnType(field.`type`().typeId())
}
diff --git
a/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala
b/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala
index cb630a1475..170be9746b 100644
---
a/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala
+++
b/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala
@@ -489,4 +489,30 @@ abstract class IcebergSuite extends
WholeStageTransformerSuite {
}
}
}
+
+ test("test read v1 iceberg with partition drop") {
+ val testTable = "test_table_with_partition"
+ withTable(testTable) {
+ spark.sql(s"""
+ |CREATE TABLE $testTable (id INT, data STRING, p1 STRING,
p2 STRING)
+ |USING iceberg
+ |tblproperties (
+ | 'format-version' = '1'
+ |)
+ |PARTITIONED BY (p1, p2);
+ |""".stripMargin)
+ spark.sql(s"""
+ |INSERT INTO $testTable VALUES
+ |(1, 'test_data', 'test_p1', 'test_p2');
+ |""".stripMargin)
+ spark.sql(s"""
+ |ALTER TABLE $testTable DROP PARTITION FIELD p2
+ |""".stripMargin)
+ val resultDf = spark.sql(s"SELECT id, data, p1, p2 FROM $testTable")
+ val result = resultDf.collect()
+
+ assert(result.length == 1)
+ assert(result.head.getString(3) == "test_p2")
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]