anew commented on code in PR #57722:
URL: https://github.com/apache/spark/pull/57722#discussion_r3713452704
##########
sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/util/SchemaInferenceUtilsSuite.scala:
##########
@@ -318,4 +318,71 @@ class SchemaInferenceUtilsSuite extends SparkFunSuite {
assert(typeChange.head.fieldNames() === Array("value"))
assert(typeChange.head.newDataType() === LongType)
}
+
+ test("mergeSchemas - a nested field differing only in case folds onto the
existing field when " +
+ "case-insensitive") {
+ // The nested analog of the top-level case-only fold. `StructType.merge`
propagates the
+ // case-sensitivity flag into nested struct merges (SPARK-58525), so the
incoming `s.Value` is
+ // matched to the existing `s.value` and the struct keeps a single field
with the persisted
+ // (left) spelling -- rather than growing a second, case-differing nested
field.
+ val currentSchema = new StructType()
+ .add("id", IntegerType)
+ .add("s", new StructType().add("value", StringType))
+ val dataSchema = new StructType()
+ .add("id", IntegerType)
+ .add("s", new StructType().add("Value", StringType))
+
+ val merged =
+ SchemaMergingUtils.mergeSchemas(currentSchema, dataSchema, caseSensitive
= false)
+ assert(merged === currentSchema)
+
+ // Because the merge is a no-op, evolution derives no table changes at
all: in particular the
+ // nested struct is NOT rewritten (which would be an UpdateColumnType on
`s`).
+ assert(
+ SchemaInferenceUtils.diffSchemas(currentSchema, merged, caseSensitive =
false).isEmpty)
+ }
+
+ test("mergeSchemas - a nested field differing only in case stays distinct
when case-sensitive") {
+ // The case-sensitive control: `s.value` and `s.Value` are different
fields, so the merged
+ // struct carries both.
+ val currentSchema = new StructType().add("s", new
StructType().add("value", StringType))
+ val dataSchema = new StructType().add("s", new StructType().add("Value",
StringType))
+
+ val merged = SchemaMergingUtils.mergeSchemas(currentSchema, dataSchema,
caseSensitive = true)
+ assert(
+ merged === new StructType().add(
+ "s",
+ new StructType().add("value", StringType).add("Value", StringType)))
+ }
Review Comment:
can we add an assertion for diffSchemas here, similar to the previous test?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
