dongjoon-hyun commented on a change in pull request #24260: [SPARK-27329][SQL]
Pruning nested field in map of map key and value from object serializers
URL: https://github.com/apache/spark/pull/24260#discussion_r270697218
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala
##########
@@ -175,30 +175,24 @@ object ObjectSerializerPruning extends Rule[LogicalPlan]
{
serializer: NamedExpression,
prunedDataType: DataType): NamedExpression = {
val prunedStructTypes = collectStructType(prunedDataType,
ArrayBuffer.empty[StructType])
- var structTypeIndex = 0
+ .toIterator
- val transformedSerializer = serializer.transformDown {
+ def transformer: PartialFunction[Expression, Expression] = {
case m: ExternalMapToCatalyst =>
- val prunedKeyConverter = m.keyConverter.transformDown {
- case s: CreateNamedStruct if structTypeIndex <
prunedStructTypes.size =>
- val prunedType = prunedStructTypes(structTypeIndex)
- structTypeIndex += 1
- pruneNamedStruct(s, prunedType)
- }
- val prunedValueConverter = m.valueConverter.transformDown {
- case s: CreateNamedStruct if structTypeIndex <
prunedStructTypes.size =>
- val prunedType = prunedStructTypes(structTypeIndex)
- structTypeIndex += 1
- pruneNamedStruct(s, prunedType)
- }
+ val prunedKeyConverter = m.keyConverter.transformDown(transformer)
+ val prunedValueConverter = m.valueConverter.transformDown(transformer)
+
m.copy(keyConverter = alignNullTypeInIf(prunedKeyConverter),
valueConverter = alignNullTypeInIf(prunedValueConverter))
- case s: CreateNamedStruct if structTypeIndex < prunedStructTypes.size =>
- val prunedType = prunedStructTypes(structTypeIndex)
- structTypeIndex += 1
+
+ case s: CreateNamedStruct if prunedStructTypes.hasNext =>
+ val prunedType = prunedStructTypes.next()
pruneNamedStruct(s, prunedType)
+
+ case other => other
Review comment:
Since this `PartialFunction` is passed to `transformDown`, we don't need
this `case other => other`. Please remove line 191 ~ 192.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
