Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/20911#discussion_r178241833
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
---
@@ -327,18 +314,69 @@ class ComplexTypesSuite extends PlanTest with
ExpressionEvalHelper {
// but it cannot override a potential match with ('id + 2L),
// which is exactly what [[Coalesce]] would do in this case.
(Literal.TrueLiteral, 'id))) as "a")
- .analyze
- comparePlans(Optimizer execute rel, expected)
+ checkRule(rel, expected)
+ }
+
+ test("SPARK-23500: Simplify array ops that are not at the top node") {
+ val query = LocalRelation('id.long)
+ .select(
+ CreateArray(Seq(
+ CreateNamedStruct(Seq(
+ "att1", 'id,
+ "att2", 'id * 'id)),
+ CreateNamedStruct(Seq(
+ "att1", 'id + 1,
+ "att2", ('id + 1) * ('id + 1))
+ ))
+ ) as "arr")
+ .select(
+ GetStructField(GetArrayItem('arr, 1), 0, None) as "a1",
+ GetArrayItem(
+ GetArrayStructFields('arr,
+ StructField("att1", LongType, nullable = false),
+ ordinal = 0,
+ numFields = 1,
+ containsNull = false),
+ ordinal = 1) as "a2")
+ .orderBy('id.asc)
+
+ val expected = LocalRelation('id.long)
+ .select(
+ ('id + 1L) as "a1",
+ ('id + 1L) as "a2")
+ .orderBy('id.asc)
+ checkRule(query, expected)
+ }
+
+ test("SPARK-23500: Simplify map ops that are not top nodes") {
+ val query =
+ LocalRelation('id.long)
+ .select(
+ CreateMap(Seq(
+ "r1", 'id,
+ "r2", 'id + 1L)) as "m")
+ .select(
+ GetMapValue('m, "r1") as "a1",
+ GetMapValue('m, "r32") as "a2")
+ .where('id > 0L)
--- End diff --
can we use sort here too?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
