beliefer commented on code in PR #43722:
URL: https://github.com/apache/spark/pull/43722#discussion_r1391022479
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala:
##########
@@ -406,10 +443,16 @@ private[sql] class XmlInferSchema(options: XmlOptions)
extends Serializable with
}
case (StructType(fields1), StructType(fields2)) =>
- val newFields = (fields1 ++ fields2).groupBy(_.name).map {
- case (name, fieldTypes) =>
+ val newFields = (fields1 ++ fields2)
+ // normalize field name and pair it with original field
+ .map(field => (normalize(field.name), field))
+ .groupBy(_._1) // group by normalized field name
+ .map {
+ case (_: String, fields: Array[(String, StructField)]) =>
Review Comment:
```suggestion
.map { case (_: String, fields: Array[(String, StructField)]) =>
```
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/XmlInferSchema.scala:
##########
@@ -406,10 +443,16 @@ private[sql] class XmlInferSchema(options: XmlOptions)
extends Serializable with
}
case (StructType(fields1), StructType(fields2)) =>
- val newFields = (fields1 ++ fields2).groupBy(_.name).map {
- case (name, fieldTypes) =>
+ val newFields = (fields1 ++ fields2)
+ // normalize field name and pair it with original field
+ .map(field => (normalize(field.name), field))
+ .groupBy(_._1) // group by normalized field name
+ .map {
+ case (_: String, fields: Array[(String, StructField)]) =>
+ val fieldTypes = fields.map(_._2)
val dataType = fieldTypes.map(_.dataType).reduce(compatibleType)
- StructField(name, dataType, nullable = true)
+ // we pick up the first field name that we've encountered for
the field
+ StructField(fields(0)._2.name, dataType)
Review Comment:
```suggestion
StructField(fields.head._2.name, dataType)
```
##########
sql/core/src/test/resources/test-data/xml-resources/attributes-case-sensitive.xml:
##########
@@ -0,0 +1,12 @@
+<?xml version="1.0"?>
+<ROWSET>
+ <ROW>
+ <struct attr1="1">1</struct>
+ <array attr2="2">2</array>
+ <array Attr2="3">3</array>
+ <array aTTr2="4">4</array>
Review Comment:
The array element looks confused. cc @sandip-db
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
