maropu commented on a change in pull request #30443:
URL: https://github.com/apache/spark/pull/30443#discussion_r545609783
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
##########
@@ -452,6 +453,16 @@ package object dsl {
def hint(name: String, parameters: Any*): LogicalPlan =
UnresolvedHint(name, parameters, logicalPlan)
+
+ def sample(
+ lowerBound: Double = 0.0,
+ upperBound: Double,
+ withReplacement: Boolean = false,
+ seed: Long = Utils.random.nextLong): LogicalPlan = {
Review comment:
Why do some params above have default values?
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
##########
@@ -541,6 +561,14 @@ case class Range(
s"Range ($start, $end, step=$step, splits=$numSlices)"
}
+ override def maxRows: Option[Long] = {
+ if (numElements.isValidLong) {
+ Some(numElements.toLong)
+ } else {
+ None
Review comment:
Could you add tests for this code path?
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
##########
@@ -958,6 +987,7 @@ case class Distinct(child: LogicalPlan) extends UnaryNode {
abstract class RepartitionOperation extends UnaryNode {
def shuffle: Boolean
def numPartitions: Int
+ override def maxRows: Option[Long] = child.maxRows
Review comment:
nit: could we add `final` for this?
##########
File path:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombiningLimitsSuite.scala
##########
@@ -117,4 +130,85 @@ class CombiningLimitsSuite extends PlanTest {
testRelation.select().groupBy()(count(1)).orderBy(count(1).asc).analyze)
comparePlans(optimized4, expected4)
}
+
+ test("SPARK-33497: Override maxRows in some LogicalPlan") {
+ def checkPlan(p1: LogicalPlan, p2: LogicalPlan): Unit = {
+ comparePlans(Optimize.execute(p1.analyze), p2.analyze)
+ }
+
+ // test LocalRelation
+ checkPlan(
+ testRelation.select().limit(10),
+ testRelation.select()
+ )
+
+ // test Range
+ checkPlan(
+ Range(1, 100, 1, None).select().limit(200),
+ Range(1, 100, 1, None).select()
+ )
+
+ // test Sample
+ val sampleQuery = testRelation.select().sample(upperBound =
0.2).limit(10).analyze
+ val sampleOptimized = Optimize.execute(sampleQuery)
+ assert(sampleOptimized.collect { case l @ Limit(_, _) => l }.isEmpty)
Review comment:
How about passing a `seed` param into `sample` and then using
`checkPlan`?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
