cloud-fan commented on a change in pull request #30443:
URL: https://github.com/apache/spark/pull/30443#discussion_r547252997
##########
File path:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombiningLimitsSuite.scala
##########
@@ -117,4 +128,118 @@ class CombiningLimitsSuite extends PlanTest {
testRelation.select().groupBy()(count(1)).orderBy(count(1).asc).analyze)
comparePlans(optimized4, expected4)
}
+
+ test("SPARK-33497: Eliminate Limit if LocalRelation max rows not larger than
Limit") {
+ checkPlanAndMaxRow(
+ testRelation.select().limit(10),
+ testRelation.select(),
+ 10
+ )
+ }
+
+ test("SPARK-33497: Eliminate Limit if Range max rows not larger than Limit")
{
+ checkPlanAndMaxRow(
+ Range(0, 100, 1, None).select().limit(200),
+ Range(0, 100, 1, None).select(),
+ 100
+ )
+ checkPlanAndMaxRow(
+ Range(-1, Long.MaxValue, 1, None).select().limit(1),
+ Range(-1, Long.MaxValue, 1, None).select().limit(1),
+ 1
+ )
+ }
+
+ test("SPARK-33497: Eliminate Limit if Sample max rows not larger than
Limit") {
+ checkPlanAndMaxRow(
+ testRelation.select().sample(upperBound = 0.2, seed = 1).limit(10),
+ testRelation.select().sample(upperBound = 0.2, seed = 1),
+ 10
+ )
+ }
+
+ test("SPARK-33497: Eliminate Limit if Deduplicate max rows not larger than
Limit") {
+ checkPlanAndMaxRow(
+ testRelation.deduplicate(Symbol("a")).limit(10),
+ testRelation.deduplicate(Symbol("a")),
Review comment:
I just found out a better way: `"a".attr`. Can you make a try and see if
it can replace `Symbol("a")`?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
