This is an automated email from the ASF dual-hosted git repository.
ptoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c3b102bf3ea4 [SPARK-54304][SQL][TESTS] MergeScalarSubqueriesSuite code
cleanup
c3b102bf3ea4 is described below
commit c3b102bf3ea4a1b2a48a99068c5bbf436a373544
Author: Peter Toth <[email protected]>
AuthorDate: Tue Nov 11 18:43:46 2025 +0100
[SPARK-54304][SQL][TESTS] MergeScalarSubqueriesSuite code cleanup
### What changes were proposed in this pull request?
This is minor code cleaup in `MergeScalarSubqueriesSuite`.
### Why are the changes needed?
No change in tests just better readability before adding new ones.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing UTs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #52998 from
peter-toth/SPARK-54304-mergescalarsubqueriessuite-code-cleanup.
Authored-by: Peter Toth <[email protected]>
Signed-off-by: Peter Toth <[email protected]>
---
.../optimizer/MergeScalarSubqueriesSuite.scala | 305 ++++++++++-----------
1 file changed, 146 insertions(+), 159 deletions(-)
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/MergeScalarSubqueriesSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/MergeScalarSubqueriesSuite.scala
index c167b80c1827..008b4a89ce60 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/MergeScalarSubqueriesSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/MergeScalarSubqueriesSuite.scala
@@ -35,28 +35,33 @@ class MergeScalarSubqueriesSuite extends PlanTest {
val batches = Batch("MergeScalarSubqueries", Once, MergeScalarSubqueries)
:: Nil
}
- val testRelation = LocalRelation(Symbol("a").int, Symbol("b").int,
Symbol("c").string)
+ val testRelation = LocalRelation($"a".int, $"b".int, $"c".string)
val testRelationWithNonBinaryCollation = LocalRelation(
- Symbol("utf8_binary").string("UTF8_BINARY"),
- Symbol("utf8_lcase").string("UTF8_LCASE"))
+ $"utf8_binary".string("UTF8_BINARY"),
+ $"utf8_lcase".string("UTF8_LCASE"))
private def definitionNode(plan: LogicalPlan, cteIndex: Int) = {
CTERelationDef(plan, cteIndex, underSubquery = true)
}
- private def extractorExpression(cteIndex: Int, output: Seq[Attribute],
fieldIndex: Int) = {
- GetStructField(ScalarSubquery(
- CTERelationRef(cteIndex, _resolved = true, output, isStreaming =
false)), fieldIndex)
- .as("scalarsubquery()")
+ private def extractorExpression(
+ cteIndex: Int,
+ output: Seq[Attribute],
+ fieldIndex: Int,
+ alias: String = "scalarsubquery()") = {
+ GetStructField(
+ ScalarSubquery(CTERelationRef(cteIndex, _resolved = true, output,
isStreaming = false)),
+ fieldIndex)
+ .as(alias)
}
test("Merging subqueries with projects") {
- val subquery1 = ScalarSubquery(testRelation.select((Symbol("a") +
1).as("a_plus1")))
- val subquery2 = ScalarSubquery(testRelation.select((Symbol("a") +
2).as("a_plus2")))
- val subquery3 = ScalarSubquery(testRelation.select(Symbol("b")))
- val subquery4 = ScalarSubquery(testRelation.select((Symbol("a") +
1).as("a_plus1_2")))
- val subquery5 = ScalarSubquery(testRelation.select((Symbol("a") +
2).as("a_plus2_2")))
- val subquery6 = ScalarSubquery(testRelation.select(Symbol("b").as("b_2")))
+ val subquery1 = ScalarSubquery(testRelation.select(($"a" +
1).as("a_plus1")))
+ val subquery2 = ScalarSubquery(testRelation.select(($"a" +
2).as("a_plus2")))
+ val subquery3 = ScalarSubquery(testRelation.select($"b"))
+ val subquery4 = ScalarSubquery(testRelation.select(($"a" +
1).as("a_plus1_2")))
+ val subquery5 = ScalarSubquery(testRelation.select(($"a" +
2).as("a_plus2_2")))
+ val subquery6 = ScalarSubquery(testRelation.select($"b".as("b_2")))
val originalQuery = testRelation
.select(
subquery1,
@@ -68,14 +73,14 @@ class MergeScalarSubqueriesSuite extends PlanTest {
val mergedSubquery = testRelation
.select(
- (Symbol("a") + 1).as("a_plus1"),
- (Symbol("a") + 2).as("a_plus2"),
- Symbol("b"))
+ ($"a" + 1).as("a_plus1"),
+ ($"a" + 2).as("a_plus2"),
+ $"b")
.select(
CreateNamedStruct(Seq(
- Literal("a_plus1"), Symbol("a_plus1"),
- Literal("a_plus2"), Symbol("a_plus2"),
- Literal("b"), Symbol("b")
+ Literal("a_plus1"), $"a_plus1",
+ Literal("a_plus2"), $"a_plus2",
+ Literal("b"), $"b"
)).as("mergedValue"))
val analyzedMergedSubquery = mergedSubquery.analyze
val correctAnswer = WithCTE(
@@ -93,14 +98,12 @@ class MergeScalarSubqueriesSuite extends PlanTest {
}
test("Merging subqueries with aggregates") {
- val subquery1 =
ScalarSubquery(testRelation.groupBy(Symbol("b"))(max(Symbol("a")).as("max_a")))
- val subquery2 =
ScalarSubquery(testRelation.groupBy(Symbol("b"))(sum(Symbol("a")).as("sum_a")))
- val subquery3 =
ScalarSubquery(testRelation.groupBy(Symbol("b"))(Symbol("b")))
- val subquery4 = ScalarSubquery(
- testRelation.groupBy(Symbol("b"))(max(Symbol("a")).as("max_a_2")))
- val subquery5 = ScalarSubquery(
- testRelation.groupBy(Symbol("b"))(sum(Symbol("a")).as("sum_a_2")))
- val subquery6 =
ScalarSubquery(testRelation.groupBy(Symbol("b"))(Symbol("b").as("b_2")))
+ val subquery1 =
ScalarSubquery(testRelation.groupBy($"b")(max($"a").as("max_a")))
+ val subquery2 =
ScalarSubquery(testRelation.groupBy($"b")(sum($"a").as("sum_a")))
+ val subquery3 = ScalarSubquery(testRelation.groupBy($"b")($"b"))
+ val subquery4 =
ScalarSubquery(testRelation.groupBy($"b")(max($"a").as("max_a_2")))
+ val subquery5 =
ScalarSubquery(testRelation.groupBy($"b")(sum($"a").as("sum_a_2")))
+ val subquery6 = ScalarSubquery(testRelation.groupBy($"b")($"b".as("b_2")))
val originalQuery = testRelation
.select(
subquery1,
@@ -111,14 +114,14 @@ class MergeScalarSubqueriesSuite extends PlanTest {
subquery6)
val mergedSubquery = testRelation
- .groupBy(Symbol("b"))(
- max(Symbol("a")).as("max_a"),
- sum(Symbol("a")).as("sum_a"),
- Symbol("b"))
+ .groupBy($"b")(
+ max($"a").as("max_a"),
+ sum($"a").as("sum_a"),
+ $"b")
.select(CreateNamedStruct(Seq(
- Literal("max_a"), Symbol("max_a"),
- Literal("sum_a"), Symbol("sum_a"),
- Literal("b"), Symbol("b")
+ Literal("max_a"), $"max_a",
+ Literal("sum_a"), $"sum_a",
+ Literal("b"), $"b"
)).as("mergedValue"))
val analyzedMergedSubquery = mergedSubquery.analyze
val correctAnswer = WithCTE(
@@ -136,13 +139,11 @@ class MergeScalarSubqueriesSuite extends PlanTest {
}
test("Merging subqueries with aggregates with complex grouping expressions")
{
- val subquery1 = ScalarSubquery(testRelation.groupBy(
- Symbol("b") > 1 && Symbol("a") === 2)(max(Symbol("a")).as("max_a")))
- val subquery2 = ScalarSubquery(
- testRelation
- .select(Symbol("a"), Symbol("b").as("b_2"))
- .groupBy(Literal(2) === Symbol("a") &&
- Literal(1) < Symbol("b_2"))(sum(Symbol("a")).as("sum_a")))
+ val subquery1 = ScalarSubquery(
+ testRelation.groupBy($"b" > 1 && $"a" === 2)(max($"a").as("max_a")))
+ val subquery2 = ScalarSubquery(testRelation
+ .select($"a", $"b".as("b_2"))
+ .groupBy(Literal(2) === $"a" && Literal(1) <
$"b_2")(sum($"a").as("sum_a")))
val originalQuery = testRelation
.select(
@@ -150,13 +151,13 @@ class MergeScalarSubqueriesSuite extends PlanTest {
subquery2)
val mergedSubquery = testRelation
- .select(Symbol("a"), Symbol("b"), Symbol("c"))
- .groupBy(Symbol("b") > 1 && Symbol("a") === 2)(
- max(Symbol("a")).as("max_a"),
- sum(Symbol("a")).as("sum_a"))
+ .select($"a", $"b", $"c")
+ .groupBy($"b" > 1 && $"a" === 2)(
+ max($"a").as("max_a"),
+ sum($"a").as("sum_a"))
.select(CreateNamedStruct(Seq(
- Literal("max_a"), Symbol("max_a"),
- Literal("sum_a"), Symbol("sum_a")
+ Literal("max_a"), $"max_a",
+ Literal("sum_a"), $"sum_a"
)).as("mergedValue"))
val analyzedMergedSubquery = mergedSubquery.analyze
val correctAnswer = WithCTE(
@@ -171,10 +172,8 @@ class MergeScalarSubqueriesSuite extends PlanTest {
test("Merging subqueries with aggregates with multiple grouping
expressions") {
// supports HashAggregate
- val subquery1 = ScalarSubquery(testRelation.groupBy(Symbol("b"),
- Symbol("c"))(max(Symbol("a")).as("max_a")))
- val subquery2 = ScalarSubquery(testRelation.groupBy(Symbol("b"),
- Symbol("c"))(min(Symbol("a")).as("min_a")))
+ val subquery1 = ScalarSubquery(testRelation.groupBy($"b",
$"c")(max($"a").as("max_a")))
+ val subquery2 = ScalarSubquery(testRelation.groupBy($"b",
$"c")(min($"a").as("min_a")))
val originalQuery = testRelation
.select(
@@ -182,12 +181,12 @@ class MergeScalarSubqueriesSuite extends PlanTest {
subquery2)
val hashAggregates = testRelation
- .groupBy(Symbol("b"), Symbol("c"))(
- max(Symbol("a")).as("max_a"),
- min(Symbol("a")).as("min_a"))
+ .groupBy($"b", $"c")(
+ max($"a").as("max_a"),
+ min($"a").as("min_a"))
.select(CreateNamedStruct(Seq(
- Literal("max_a"), Symbol("max_a"),
- Literal("min_a"), Symbol("min_a")
+ Literal("max_a"), $"max_a",
+ Literal("min_a"), $"min_a"
)).as("mergedValue"))
val analyzedHashAggregates = hashAggregates.analyze
val correctAnswer = WithCTE(
@@ -201,10 +200,10 @@ class MergeScalarSubqueriesSuite extends PlanTest {
}
test("Avoid merge when mixing sort and hash aggs") {
- val subquery1 = ScalarSubquery(testRelationWithNonBinaryCollation.groupBy(
- Symbol("utf8_binary"))(max(Symbol("utf8_binary")).as("max_utf8_binary")))
- val subquery2 = ScalarSubquery(testRelationWithNonBinaryCollation.groupBy(
- Symbol("utf8_lcase"))(max(Symbol("utf8_lcase")).as("utf8_lcase")))
+ val subquery1 = ScalarSubquery(testRelationWithNonBinaryCollation
+ .groupBy($"utf8_binary")(max($"utf8_binary").as("max_utf8_binary")))
+ val subquery2 = ScalarSubquery(testRelationWithNonBinaryCollation
+ .groupBy($"utf8_lcase")(max($"utf8_lcase").as("utf8_lcase")))
val originalQuery = testRelationWithNonBinaryCollation.select(subquery1,
subquery2)
Optimize.execute(originalQuery.analyze).collect {
case WithCTE(_, _) => fail("Should not have merged")
@@ -212,16 +211,16 @@ class MergeScalarSubqueriesSuite extends PlanTest {
}
test("Merging subqueries with filters") {
- val subquery1 = ScalarSubquery(testRelation.where(Symbol("a") >
1).select(Symbol("a")))
+ val subquery1 = ScalarSubquery(testRelation.where($"a" > 1).select($"a"))
// Despite having an extra Project node, `subquery2` is mergeable with
`subquery1`
- val subquery2 = ScalarSubquery(testRelation.where(Symbol("a") > 1).select(
- Symbol("b").as("b_1")).select(Symbol("b_1")))
+ val subquery2 = ScalarSubquery(
+ testRelation.where($"a" > 1).select($"b".as("b_1")).select($"b_1"))
// Despite lacking a Project node, `subquery3` is mergeable with the
result of merging
// `subquery1` and `subquery2`
- val subquery3 = ScalarSubquery(testRelation.select(
- Symbol("a").as("a_2")).where(Symbol("a_2") > 1).select(Symbol("a_2")))
- val subquery4 = ScalarSubquery(testRelation.select(
- Symbol("a").as("a_2"), Symbol("b")).where(Symbol("a_2") >
1).select(Symbol("b").as("b_2")))
+ val subquery3 = ScalarSubquery(
+ testRelation.select($"a".as("a_2")).where($"a_2" > 1).select($"a_2"))
+ val subquery4 = ScalarSubquery(
+ testRelation.select($"a".as("a_2"), $"b").where($"a_2" >
1).select($"b".as("b_2")))
val originalQuery = testRelation
.select(
subquery1,
@@ -230,13 +229,13 @@ class MergeScalarSubqueriesSuite extends PlanTest {
subquery4)
val mergedSubquery = testRelation
- .select(Symbol("a"), Symbol("b"), Symbol("c"))
- .where(Symbol("a") > 1)
- .select(Symbol("a"), Symbol("b"), Symbol("c"))
- .select(Symbol("a"), Symbol("b"))
+ .select($"a", $"b", $"c")
+ .where($"a" > 1)
+ .select($"a", $"b", $"c")
+ .select($"a", $"b")
.select(CreateNamedStruct(Seq(
- Literal("a"), Symbol("a"),
- Literal("b"), Symbol("b")
+ Literal("a"), $"a",
+ Literal("b"), $"b"
)).as("mergedValue"))
val analyzedMergedSubquery = mergedSubquery.analyze
val correctAnswer = WithCTE(
@@ -252,25 +251,23 @@ class MergeScalarSubqueriesSuite extends PlanTest {
}
test("Merging subqueries with complex filter conditions") {
- val subquery1 = ScalarSubquery(
- testRelation.where(Symbol("a") > 1 && Symbol("b") ===
2).select(Symbol("a")))
- val subquery2 = ScalarSubquery(
- testRelation
- .select(Symbol("a").as("a_2"), Symbol("b"))
- .where(Literal(2) === Symbol("b") && Literal(1) < Symbol("a_2"))
- .select(Symbol("b").as("b_2")))
+ val subquery1 = ScalarSubquery(testRelation.where($"a" > 1 && $"b" ===
2).select($"a"))
+ val subquery2 = ScalarSubquery(testRelation
+ .select($"a".as("a_2"), $"b")
+ .where(Literal(2) === $"b" && Literal(1) < $"a_2")
+ .select($"b".as("b_2")))
val originalQuery = testRelation
.select(
subquery1,
subquery2)
val mergedSubquery = testRelation
- .select(Symbol("a"), Symbol("b"), Symbol("c"))
- .where(Symbol("a") > 1 && Symbol("b") === 2)
- .select(Symbol("a"), Symbol("b").as("b_2"))
+ .select($"a", $"b", $"c")
+ .where($"a" > 1 && $"b" === 2)
+ .select($"a", $"b".as("b_2"))
.select(CreateNamedStruct(Seq(
- Literal("a"), Symbol("a"),
- Literal("b_2"), Symbol("b_2")
+ Literal("a"), $"a",
+ Literal("b_2"), $"b_2"
)).as("mergedValue"))
val analyzedMergedSubquery = mergedSubquery.analyze
val correctAnswer = WithCTE(
@@ -284,8 +281,8 @@ class MergeScalarSubqueriesSuite extends PlanTest {
}
test("Do not merge subqueries with different filter conditions") {
- val subquery1 = ScalarSubquery(testRelation.where(Symbol("a") >
1).select(Symbol("a")))
- val subquery2 = ScalarSubquery(testRelation.where(Symbol("a") <
1).select(Symbol("a")))
+ val subquery1 = ScalarSubquery(testRelation.where($"a" > 1).select($"a"))
+ val subquery2 = ScalarSubquery(testRelation.where($"a" < 1).select($"a"))
val originalQuery = testRelation
.select(
@@ -297,23 +294,23 @@ class MergeScalarSubqueriesSuite extends PlanTest {
test("Merging subqueries with aggregate filters") {
val subquery1 = ScalarSubquery(
-
testRelation.having(Symbol("b"))(max(Symbol("a")).as("max_a"))(max(Symbol("a"))
> 1))
+ testRelation.having($"b")(max($"a").as("max_a"))(max($"a") > 1))
val subquery2 = ScalarSubquery(
-
testRelation.having(Symbol("b"))(sum(Symbol("a")).as("sum_a"))(max(Symbol("a"))
> 1))
+ testRelation.having($"b")(sum($"a").as("sum_a"))(max($"a") > 1))
val originalQuery = testRelation.select(
subquery1,
subquery2)
val mergedSubquery = testRelation
- .having(Symbol("b"))(
- max(Symbol("a")).as("max_a"),
- sum(Symbol("a")).as("sum_a"))(Symbol("max_a") > 1)
+ .having($"b")(
+ max($"a").as("max_a"),
+ sum($"a").as("sum_a"))($"max_a" > 1)
.select(
- Symbol("max_a"),
- Symbol("sum_a"))
+ $"max_a",
+ $"sum_a")
.select(CreateNamedStruct(Seq(
- Literal("max_a"), Symbol("max_a"),
- Literal("sum_a"), Symbol("sum_a")
+ Literal("max_a"), $"max_a",
+ Literal("sum_a"), $"sum_a"
)).as("mergedValue"))
val analyzedMergedSubquery = mergedSubquery.analyze
val correctAnswer = WithCTE(
@@ -334,13 +331,12 @@ class MergeScalarSubqueriesSuite extends PlanTest {
Some($"t1.b" === $"t2.b"))
.select($"t1.a").analyze)
val subquery2 = ScalarSubquery(testRelation.as("t1")
- .select(Symbol("a").as("a_1"), Symbol("b").as("b_1"),
Symbol("c").as("c_1"))
+ .select($"a".as("a_1"), $"b".as("b_1"), $"c".as("c_1"))
.join(
- testRelation.as("t2").select(Symbol("a").as("a_2"),
Symbol("b").as("b_2"),
- Symbol("c").as("c_2")),
+ testRelation.as("t2").select($"a".as("a_2"), $"b".as("b_2"),
$"c".as("c_2")),
Inner,
- Some(Symbol("b_1") === Symbol("b_2")))
- .select(Symbol("c_2")).analyze)
+ Some($"b_1" === $"b_2"))
+ .select($"c_2").analyze)
val originalQuery = testRelation.select(
subquery1,
subquery2)
@@ -349,8 +345,8 @@ class MergeScalarSubqueriesSuite extends PlanTest {
.join(testRelation.as("t2"), Inner, Some($"t1.b" === $"t2.b"))
.select($"t1.a", $"t2.c")
.select(CreateNamedStruct(Seq(
- Literal("a"), Symbol("a"),
- Literal("c"), Symbol("c")
+ Literal("a"), $"a",
+ Literal("c"), $"c"
)).as("mergedValue"))
val analyzedMergedSubquery = mergedSubquery.analyze
val correctAnswer = WithCTE(
@@ -371,13 +367,12 @@ class MergeScalarSubqueriesSuite extends PlanTest {
Some($"t1.b" < $"t2.b" && $"t1.a" === $"t2.c"))
.select($"t1.a").analyze)
val subquery2 = ScalarSubquery(testRelation.as("t1")
- .select(Symbol("a").as("a_1"), Symbol("b").as("b_1"),
Symbol("c").as("c_1"))
+ .select($"a".as("a_1"), $"b".as("b_1"), $"c".as("c_1"))
.join(
- testRelation.as("t2").select(Symbol("a").as("a_2"),
Symbol("b").as("b_2"),
- Symbol("c").as("c_2")),
+ testRelation.as("t2").select($"a".as("a_2"), $"b".as("b_2"),
$"c".as("c_2")),
Inner,
- Some(Symbol("c_2") === Symbol("a_1") && Symbol("b_1") < Symbol("b_2")))
- .select(Symbol("c_2")).analyze)
+ Some($"c_2" === $"a_1" && $"b_1" < $"b_2"))
+ .select($"c_2").analyze)
val originalQuery = testRelation.select(
subquery1,
subquery2)
@@ -386,8 +381,8 @@ class MergeScalarSubqueriesSuite extends PlanTest {
.join(testRelation.as("t2"), Inner, Some($"t1.b" < $"t2.b" && $"t1.a"
=== $"t2.c"))
.select($"t1.a", $"t2.c")
.select(CreateNamedStruct(Seq(
- Literal("a"), Symbol("a"),
- Literal("c"), Symbol("c")
+ Literal("a"), $"a",
+ Literal("c"), $"c"
)).as("mergedValue"))
val analyzedMergedSubquery = mergedSubquery.analyze
val correctAnswer = WithCTE(
@@ -441,10 +436,8 @@ class MergeScalarSubqueriesSuite extends PlanTest {
}
test("Do not merge subqueries with nondeterministic elements") {
- val subquery1 = ScalarSubquery(
- testRelation.select((Symbol("a") + rand(0)).as("rand_a")))
- val subquery2 = ScalarSubquery(
- testRelation.select((Symbol("b") + rand(0)).as("rand_b")))
+ val subquery1 = ScalarSubquery(testRelation.select(($"a" +
rand(0)).as("rand_a")))
+ val subquery2 = ScalarSubquery(testRelation.select(($"b" +
rand(0)).as("rand_b")))
val originalQuery = testRelation
.select(
subquery1,
@@ -452,10 +445,8 @@ class MergeScalarSubqueriesSuite extends PlanTest {
comparePlans(Optimize.execute(originalQuery.analyze),
originalQuery.analyze)
- val subquery3 = ScalarSubquery(
- testRelation.where(Symbol("a") > rand(0)).select(Symbol("a")))
- val subquery4 = ScalarSubquery(
- testRelation.where(Symbol("a") > rand(0)).select(Symbol("b")))
+ val subquery3 = ScalarSubquery(testRelation.where($"a" >
rand(0)).select($"a"))
+ val subquery4 = ScalarSubquery(testRelation.where($"a" >
rand(0)).select($"b"))
val originalQuery2 = testRelation
.select(
subquery3,
@@ -463,10 +454,8 @@ class MergeScalarSubqueriesSuite extends PlanTest {
comparePlans(Optimize.execute(originalQuery2.analyze),
originalQuery2.analyze)
- val subquery5 = ScalarSubquery(
- testRelation.groupBy()((max(Symbol("a")) + rand(0)).as("max_a")))
- val subquery6 = ScalarSubquery(
- testRelation.groupBy()((max(Symbol("b")) + rand(0)).as("max_b")))
+ val subquery5 = ScalarSubquery(testRelation.groupBy()((max($"a") +
rand(0)).as("max_a")))
+ val subquery6 = ScalarSubquery(testRelation.groupBy()((max($"b") +
rand(0)).as("max_b")))
val originalQuery3 = testRelation
.select(
subquery5,
@@ -477,20 +466,20 @@ class MergeScalarSubqueriesSuite extends PlanTest {
test("Do not merge different aggregate implementations") {
// supports HashAggregate
- val subquery1 =
ScalarSubquery(testRelation.groupBy(Symbol("b"))(max(Symbol("a")).as("max_a")))
- val subquery2 =
ScalarSubquery(testRelation.groupBy(Symbol("b"))(min(Symbol("a")).as("min_a")))
+ val subquery1 =
ScalarSubquery(testRelation.groupBy($"b")(max($"a").as("max_a")))
+ val subquery2 =
ScalarSubquery(testRelation.groupBy($"b")(min($"a").as("min_a")))
// supports ObjectHashAggregate
val subquery3 = ScalarSubquery(testRelation
- .groupBy(Symbol("b"))(CollectList(Symbol("a")).
+ .groupBy($"b")(CollectList($"a").
toAggregateExpression(isDistinct = false).as("collectlist_a")))
val subquery4 = ScalarSubquery(testRelation
- .groupBy(Symbol("b"))(CollectSet(Symbol("a")).
+ .groupBy($"b")(CollectSet($"a").
toAggregateExpression(isDistinct = false).as("collectset_a")))
// supports SortAggregate
- val subquery5 =
ScalarSubquery(testRelation.groupBy(Symbol("b"))(max(Symbol("c")).as("max_c")))
- val subquery6 =
ScalarSubquery(testRelation.groupBy(Symbol("b"))(min(Symbol("c")).as("min_c")))
+ val subquery5 =
ScalarSubquery(testRelation.groupBy($"b")(max($"c").as("max_c")))
+ val subquery6 =
ScalarSubquery(testRelation.groupBy($"b")(min($"c").as("min_c")))
val originalQuery = testRelation
.select(
@@ -502,30 +491,30 @@ class MergeScalarSubqueriesSuite extends PlanTest {
subquery6)
val hashAggregates = testRelation
- .groupBy(Symbol("b"))(
- max(Symbol("a")).as("max_a"),
- min(Symbol("a")).as("min_a"))
+ .groupBy($"b")(
+ max($"a").as("max_a"),
+ min($"a").as("min_a"))
.select(CreateNamedStruct(Seq(
- Literal("max_a"), Symbol("max_a"),
- Literal("min_a"), Symbol("min_a")
+ Literal("max_a"), $"max_a",
+ Literal("min_a"), $"min_a"
)).as("mergedValue"))
val analyzedHashAggregates = hashAggregates.analyze
val objectHashAggregates = testRelation
- .groupBy(Symbol("b"))(
- CollectList(Symbol("a")).toAggregateExpression(isDistinct =
false).as("collectlist_a"),
- CollectSet(Symbol("a")).toAggregateExpression(isDistinct =
false).as("collectset_a"))
+ .groupBy($"b")(
+ CollectList($"a").toAggregateExpression(isDistinct =
false).as("collectlist_a"),
+ CollectSet($"a").toAggregateExpression(isDistinct =
false).as("collectset_a"))
.select(CreateNamedStruct(Seq(
- Literal("collectlist_a"), Symbol("collectlist_a"),
- Literal("collectset_a"), Symbol("collectset_a")
+ Literal("collectlist_a"), $"collectlist_a",
+ Literal("collectset_a"), $"collectset_a"
)).as("mergedValue"))
val analyzedObjectHashAggregates = objectHashAggregates.analyze
val sortAggregates = testRelation
- .groupBy(Symbol("b"))(
- max(Symbol("c")).as("max_c"),
- min(Symbol("c")).as("min_c"))
+ .groupBy($"b")(
+ max($"c").as("max_c"),
+ min($"c").as("min_c"))
.select(CreateNamedStruct(Seq(
- Literal("max_c"), Symbol("max_c"),
- Literal("min_c"), Symbol("min_c")
+ Literal("max_c"), $"max_c",
+ Literal("min_c"), $"min_c"
)).as("mergedValue"))
val analyzedSortAggregates = sortAggregates.analyze
val correctAnswer = WithCTE(
@@ -547,10 +536,8 @@ class MergeScalarSubqueriesSuite extends PlanTest {
test("Do not merge subqueries with different aggregate grouping orders") {
// supports HashAggregate
- val subquery1 = ScalarSubquery(
- testRelation.groupBy(Symbol("b"),
Symbol("c"))(max(Symbol("a")).as("max_a")))
- val subquery2 = ScalarSubquery(
- testRelation.groupBy(Symbol("c"),
Symbol("b"))(min(Symbol("a")).as("min_a")))
+ val subquery1 = ScalarSubquery(testRelation.groupBy($"b",
$"c")(max($"a").as("max_a")))
+ val subquery2 = ScalarSubquery(testRelation.groupBy($"c",
$"b")(min($"a").as("min_a")))
val originalQuery = testRelation
.select(
@@ -561,12 +548,12 @@ class MergeScalarSubqueriesSuite extends PlanTest {
}
test("Merging subqueries from different places") {
- val subquery1 = ScalarSubquery(testRelation.select((Symbol("a") +
1).as("a_plus1")))
- val subquery2 = ScalarSubquery(testRelation.select((Symbol("a") +
2).as("a_plus2")))
- val subquery3 = ScalarSubquery(testRelation.select(Symbol("b")))
- val subquery4 = ScalarSubquery(testRelation.select((Symbol("a") +
1).as("a_plus1_2")))
- val subquery5 = ScalarSubquery(testRelation.select((Symbol("a") +
2).as("a_plus2_2")))
- val subquery6 = ScalarSubquery(testRelation.select(Symbol("b").as("b_2")))
+ val subquery1 = ScalarSubquery(testRelation.select(($"a" +
1).as("a_plus1")))
+ val subquery2 = ScalarSubquery(testRelation.select(($"a" +
2).as("a_plus2")))
+ val subquery3 = ScalarSubquery(testRelation.select($"b"))
+ val subquery4 = ScalarSubquery(testRelation.select(($"a" +
1).as("a_plus1_2")))
+ val subquery5 = ScalarSubquery(testRelation.select(($"a" +
2).as("a_plus2_2")))
+ val subquery6 = ScalarSubquery(testRelation.select($"b".as("b_2")))
val originalQuery = testRelation
.where(
subquery4 +
@@ -579,14 +566,14 @@ class MergeScalarSubqueriesSuite extends PlanTest {
val mergedSubquery = testRelation
.select(
- (Symbol("a") + 1).as("a_plus1"),
- (Symbol("a") + 2).as("a_plus2"),
- Symbol("b"))
+ ($"a" + 1).as("a_plus1"),
+ ($"a" + 2).as("a_plus2"),
+ $"b")
.select(
CreateNamedStruct(Seq(
- Literal("a_plus1"), Symbol("a_plus1"),
- Literal("a_plus2"), Symbol("a_plus2"),
- Literal("b"), Symbol("b")
+ Literal("a_plus1"), $"a_plus1",
+ Literal("a_plus2"), $"a_plus2",
+ Literal("b"), $"b"
)).as("mergedValue"))
val analyzedMergedSubquery = mergedSubquery.analyze
val correctAnswer = WithCTE(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
