cloud-fan commented on code in PR #56621:
URL: https://github.com/apache/spark/pull/56621#discussion_r3645759632
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/BasicStatsEstimationSuite.scala:
##########
@@ -193,6 +193,55 @@ test("range with invalid long value") {
checkStats(range, rangeStats, rangeStats)
}
+ test("SPARK-52163: cap estimated size in bytes to avoid BigInt overflow") {
+ val hugeAttr = attr("id")
+ // A leaf node whose estimated size is already at the maximum
representable physical size.
+ val hugePlan = StatsTestPlan(
+ outputList = Seq(hugeAttr),
+ attributeStats = AttributeMap(Seq(hugeAttr -> colStat)),
+ rowCount = BigInt(Long.MaxValue),
+ size = Some(BigInt(Long.MaxValue)))
+
+ // A cartesian join multiplies children's sizes, so without a cap the size
would explode and
+ // eventually overflow the backing BigInt when compounded across repeated
self-joins. The
+ // estimate must stay bounded and the cartesian product itself must be
capped.
+ withSQLConf(SQLConf.CBO_ENABLED.key -> "false") {
+ var node: LogicalPlan = hugePlan
+ (1 to 30).foreach { _ =>
+ val joined = node.join(node, Inner, None)
+ joined.invalidateStatsCache()
+ assert(joined.stats.sizeInBytes === BigInt(Long.MaxValue))
+ node = joined.select(hugeAttr)
+ assert(node.stats.sizeInBytes <= BigInt(Long.MaxValue))
+ }
+ }
+ }
+
+ test("SPARK-52163: cap estimated stats with CBO on to avoid BigInt
overflow") {
+ val hugeAttr = attr("id")
+ // A leaf node whose estimated size and row count are already at the
maximum physical size.
Review Comment:
`rowCount` is a cardinality rather than a physical size, so the current
wording mixes units.
```suggestion
// A leaf node whose size and row count are already at their maximum
representable values.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
