jinchengchenghh commented on code in PR #12726:
URL: https://github.com/apache/gluten/pull/12726#discussion_r4294870820
##########
gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala:
##########
@@ -742,6 +743,63 @@ abstract class IcebergSuite extends
WholeStageTransformerSuite {
}
}
+ test("case-sensitive mode: iceberg scan executes natively") {
+ // Regression test for the three unconditional .toLowerCase(Locale.ROOT)
calls that were
+ // replaced with ConverterUtils.normalizeColName in IcebergScanTransformer.
+ // Under caseSensitive=true, readSchemaFields and
inputFileRelatedMetadataColumns must
+ // preserve original casing so metadata-column detection is not confused
with data columns
+ // whose names happen to match metadata-column constants when lowercased.
+ withSQLConf("spark.sql.caseSensitive" -> "true") {
+ withTable("iceberg_cs") {
+ spark.sql("""
+ |CREATE TABLE iceberg_cs (id INT, data STRING)
Review Comment:
Looks like this is a simple test, and duplicate with other test, all the
columns are lower case
##########
gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala:
##########
@@ -742,6 +743,63 @@ abstract class IcebergSuite extends
WholeStageTransformerSuite {
}
}
+ test("case-sensitive mode: iceberg scan executes natively") {
+ // Regression test for the three unconditional .toLowerCase(Locale.ROOT)
calls that were
+ // replaced with ConverterUtils.normalizeColName in IcebergScanTransformer.
+ // Under caseSensitive=true, readSchemaFields and
inputFileRelatedMetadataColumns must
+ // preserve original casing so metadata-column detection is not confused
with data columns
+ // whose names happen to match metadata-column constants when lowercased.
+ withSQLConf("spark.sql.caseSensitive" -> "true") {
+ withTable("iceberg_cs") {
+ spark.sql("""
+ |CREATE TABLE iceberg_cs (id INT, data STRING)
+ |USING iceberg
+ |""".stripMargin)
+ spark.sql("""
+ |INSERT INTO iceberg_cs VALUES (1, 'a'), (2, 'b')
+ |""".stripMargin)
+
+ // Basic scan must use IcebergScanTransformer (not fall back to
vanilla Spark).
+ runQueryAndCompare("SELECT id, data FROM iceberg_cs ORDER BY id") {
+ checkGlutenPlan[IcebergScanTransformer]
+ }
+ }
+ }
+ }
+
+ test("case-sensitive mode: iceberg input_file_name with caseSensitive=true")
{
+ // The readSchemaFields set and inputFileRelatedMetadataColumns filter
both used
+ // unconditional locale-unaware lowercasing, which would incorrectly
classify a data
+ // column named "Input_File_Name" (mixed case) as a metadata-column
candidate because
+ // the lowercased form "input_file_name" is in
InputFileRelatedMetadataColumnNames.
+ // After the fix, ConverterUtils.normalizeColName preserves casing under
caseSensitive=true,
+ // so the lookup against the all-lowercase constant set correctly returns
false.
+ withSQLConf("spark.sql.caseSensitive" -> "true") {
+ withTable("iceberg_input_file_cs") {
+ spark.sql("""
+ |CREATE TABLE iceberg_input_file_cs (id INT, data STRING)
+ |USING iceberg
+ |""".stripMargin)
+ spark.sql("""
+ |INSERT INTO iceberg_input_file_cs VALUES (1, 'x'), (2,
'y')
+ |""".stripMargin)
+
+ // input_file_name() must work correctly under caseSensitive=true and
+ // IcebergScanTransformer must be used (no validation fallback).
+ val df = runAndCompare("""
+ |SELECT id, input_file_name() AS fname
Review Comment:
So as this one, existing test should have covered the test with lower column
names
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
