cloud-fan commented on a change in pull request #32301:
URL: https://github.com/apache/spark/pull/32301#discussion_r638939038
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
##########
@@ -227,137 +297,114 @@ object NestedColumnAliasing {
}
/**
- * This prunes unnecessary nested columns from `Generate` and optional
`Project` on top
- * of it.
+ * This prunes unnecessary nested columns from [[Generate]], or [[Project]] ->
[[Generate]]
*/
object GeneratorNestedColumnAliasing {
- // Partitions `attrToAliases` based on whether the attribute is in
Generator's output.
- private def aliasesOnGeneratorOutput(
- attrToAliases: Map[ExprId, Seq[Alias]],
- generatorOutput: Seq[Attribute]) = {
- val generatorOutputExprId = generatorOutput.map(_.exprId)
- attrToAliases.partition { k =>
- generatorOutputExprId.contains(k._1)
- }
- }
-
- // Partitions `nestedFieldToAlias` based on whether the attribute of nested
field extractor
- // is in Generator's output.
- private def nestedFieldOnGeneratorOutput(
- nestedFieldToAlias: Map[ExtractValue, Alias],
- generatorOutput: Seq[Attribute]) = {
- val generatorOutputSet = AttributeSet(generatorOutput)
- nestedFieldToAlias.partition { pair =>
- pair._1.references.subsetOf(generatorOutputSet)
- }
- }
-
def unapply(plan: LogicalPlan): Option[LogicalPlan] = plan match {
// Either `nestedPruningOnExpressions` or `nestedSchemaPruningEnabled` is
enabled, we
// need to prune nested columns through Project and under Generate. The
difference is
// when `nestedSchemaPruningEnabled` is on, nested columns will be pruned
further at
// file format readers if it is supported.
case Project(projectList, g: Generate) if
(SQLConf.get.nestedPruningOnExpressions ||
- SQLConf.get.nestedSchemaPruningEnabled) &&
canPruneGenerator(g.generator) =>
Review comment:
nit: the previous indentation is correct
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
