This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new cc8ebe2962 [GLUTEN-9481][ICEBERG] Fix issue where iceberg columns
where not properly being sanitized. (#9482)
cc8ebe2962 is described below
commit cc8ebe2962289da5d301ee927ce266a33b90d1a7
Author: Zaheen <[email protected]>
AuthorDate: Thu May 1 13:58:02 2025 -0700
[GLUTEN-9481][ICEBERG] Fix issue where iceberg columns where not properly
being sanitized. (#9482)
Properly sanitizes iceberg columns with special characters so it can be
read.
---
.../gluten/execution/IcebergScanTransformer.scala | 3 ++-
.../org/apache/gluten/execution/IcebergSuite.scala | 19 +++++++++++++++++++
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git
a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala
b/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala
index 5b15198739..6195798cc1 100644
---
a/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala
+++
b/gluten-iceberg/src/main/scala/org/apache/gluten/execution/IcebergScanTransformer.scala
@@ -32,6 +32,7 @@ import
org.apache.spark.sql.execution.datasources.v2.BatchScanExec
import org.apache.spark.sql.types.StructType
import org.apache.iceberg.{BaseTable, MetadataColumns, SnapshotSummary}
+import org.apache.iceberg.avro.AvroSchemaUtil
import org.apache.iceberg.spark.source.{GlutenIcebergSourceUtil, SparkTable}
import org.apache.iceberg.types.Type
import org.apache.iceberg.types.Type.TypeID
@@ -161,7 +162,7 @@ case class IcebergScanTransformer(
object IcebergScanTransformer {
def apply(batchScan: BatchScanExec): IcebergScanTransformer = {
new IcebergScanTransformer(
- batchScan.output,
+ batchScan.output.map(a =>
a.withName(AvroSchemaUtil.makeCompatibleName(a.name))),
batchScan.scan,
batchScan.runtimeFilters,
table = SparkShimLoader.getSparkShims.getBatchScanExecTable(batchScan),
diff --git
a/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala
b/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala
index ec071fff73..296db20829 100644
---
a/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala
+++
b/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala
@@ -645,4 +645,23 @@ abstract class IcebergSuite extends
WholeStageTransformerSuite {
assert(result.head.getString(3) == "test_p2")
}
}
+
+ test("test read iceberg with special characters in column name") {
+ val testTable = "test_table_with_special_characters"
+ withTable(testTable) {
+ spark.sql(s"""
+ |CREATE TABLE $testTable (id INT, `my/data` STRING)
+ |USING iceberg
+ |""".stripMargin)
+ spark.sql(s"""
+ |INSERT INTO $testTable VALUES
+ |(1, 'test_data');
+ |""".stripMargin)
+ val resultDf = spark.sql(s"SELECT id, `my/data` FROM $testTable")
+ val result = resultDf.collect()
+
+ assert(result.length == 1)
+ assert(result.head.getString(1) == "test_data")
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
