Github user dongjoon-hyun commented on a diff in the pull request:
https://github.com/apache/spark/pull/20687#discussion_r173520459
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
---
@@ -22,32 +22,24 @@ import
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.rules.Rule
/**
-* push down operations into [[CreateNamedStructLike]].
+* Simplify redundant [[CreateNamedStructLike]], [[CreateArray]] and
[[CreateMap]] expressions.
*/
-object SimplifyCreateStructOps extends Rule[LogicalPlan] {
- override def apply(plan: LogicalPlan): LogicalPlan = {
- plan.transformExpressionsUp {
- // push down field extraction
+object SimplifyExtractValueOps extends Rule[LogicalPlan] {
+ override def apply(plan: LogicalPlan): LogicalPlan = plan transform {
case p =>
+ p.transformExpressionsUp {
+ // Remove redundant field extraction.
case GetStructField(createNamedStructLike: CreateNamedStructLike,
ordinal, _) =>
createNamedStructLike.valExprs(ordinal)
- }
- }
-}
-/**
-* push down operations into [[CreateArray]].
-*/
-object SimplifyCreateArrayOps extends Rule[LogicalPlan] {
- override def apply(plan: LogicalPlan): LogicalPlan = {
- plan.transformExpressionsUp {
- // push down field selection (array of structs)
+ // Remove redundant array indexing.
case GetArrayStructFields(CreateArray(elems), field, ordinal,
numFields, containsNull) =>
// instead f selecting the field on the entire array,
// select it from each member of the array.
// pushing down the operation this way open other optimizations
opportunities
// (i.e. struct(...,x,...).x)
CreateArray(elems.map(GetStructField(_, ordinal,
Some(field.name))))
- // push down item selection.
+
+ // Remove redundant map lookup.
case ga @ GetArrayItem(CreateArray(elems), IntegerLiteral(idx)) =>
// instead of creating the array and then selecting one row,
// remove array creation altgether.
--- End diff --
`altgether` -> `altogether`?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
