Yaohua628 commented on a change in pull request #35147:
URL: https://github.com/apache/spark/pull/35147#discussion_r784456351
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaPruning.scala
##########
@@ -179,12 +189,16 @@ object SchemaPruning extends Rule[LogicalPlan] {
// We need to replace the expression ids of the pruned relation output
attributes
// with the expression ids of the original relation output attributes so
that
// references to the original relation's output are not broken
- val outputIdMap = output.map(att => (att.name, att.exprId)).toMap
+
+ // We also need to keep the metadata of the attribute,
+ // so that references to the metadata struct is not broken
+ val nameAttributeMap = output.map(att => (att.name, att)).toMap
requiredSchema
.toAttributes
.map {
- case att if outputIdMap.contains(att.name) =>
- att.withExprId(outputIdMap(att.name))
+ case att if nameAttributeMap.contains(att.name) =>
+ val outputAttribute = nameAttributeMap(att.name)
+
att.withExprId(outputAttribute.exprId).withMetadata(outputAttribute.metadata)
Review comment:
make sens! done!
##########
File path:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaPruning.scala
##########
@@ -179,12 +189,16 @@ object SchemaPruning extends Rule[LogicalPlan] {
// We need to replace the expression ids of the pruned relation output
attributes
// with the expression ids of the original relation output attributes so
that
// references to the original relation's output are not broken
- val outputIdMap = output.map(att => (att.name, att.exprId)).toMap
+
+ // We also need to keep the metadata of the attribute,
+ // so that references to the metadata struct is not broken
+ val nameAttributeMap = output.map(att => (att.name, att)).toMap
requiredSchema
.toAttributes
.map {
- case att if outputIdMap.contains(att.name) =>
- att.withExprId(outputIdMap(att.name))
+ case att if nameAttributeMap.contains(att.name) =>
+ val outputAttribute = nameAttributeMap(att.name)
+
att.withExprId(outputAttribute.exprId).withMetadata(outputAttribute.metadata)
Review comment:
make sense! done!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
