Github user ron8hu commented on a diff in the pull request:
https://github.com/apache/spark/pull/17148#discussion_r104527462
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala
---
@@ -254,133 +270,118 @@ class FilterEstimationSuite extends
StatsEstimationTestBase {
val d20170104 = Date.valueOf("2017-01-04")
val d20170105 = Date.valueOf("2017-01-05")
validateEstimatedStats(
- arDate,
- Filter(In(arDate, Seq(Literal(d20170103), Literal(d20170104),
Literal(d20170105))),
- childStatsTestPlan(Seq(arDate), 10L)),
- ColumnStat(distinctCount = 3, min = Some(d20170103), max =
Some(d20170105),
- nullCount = 0, avgLen = 4, maxLen = 4),
- 3)
+ Filter(In(attrDate, Seq(Literal(d20170103), Literal(d20170104),
Literal(d20170105))),
+ childStatsTestPlan(Seq(attrDate), 10L)),
+ Seq(attrDate -> ColumnStat(distinctCount = 3, min = Some(d20170103),
max = Some(d20170105),
+ nullCount = 0, avgLen = 4, maxLen = 4)),
+ expectedRowCount = 3)
}
test("cdecimal = 0.400000000000000000") {
val dec_0_40 = new java.math.BigDecimal("0.400000000000000000")
validateEstimatedStats(
- arDecimal,
- Filter(EqualTo(arDecimal, Literal(dec_0_40)),
- childStatsTestPlan(Seq(arDecimal), 4L)),
- ColumnStat(distinctCount = 1, min = Some(dec_0_40), max =
Some(dec_0_40),
- nullCount = 0, avgLen = 8, maxLen = 8),
- 1)
+ Filter(EqualTo(attrDecimal, Literal(dec_0_40)),
+ childStatsTestPlan(Seq(attrDecimal), 4L)),
+ Seq(attrDecimal -> ColumnStat(distinctCount = 1, min =
Some(dec_0_40), max = Some(dec_0_40),
+ nullCount = 0, avgLen = 8, maxLen = 8)),
+ expectedRowCount = 1)
}
test("cdecimal < 0.60 ") {
val dec_0_60 = new java.math.BigDecimal("0.600000000000000000")
validateEstimatedStats(
- arDecimal,
- Filter(LessThan(arDecimal, Literal(dec_0_60)),
- childStatsTestPlan(Seq(arDecimal), 4L)),
- ColumnStat(distinctCount = 3, min = Some(decMin), max =
Some(dec_0_60),
- nullCount = 0, avgLen = 8, maxLen = 8),
- 3)
+ Filter(LessThan(attrDecimal, Literal(dec_0_60)),
+ childStatsTestPlan(Seq(attrDecimal), 4L)),
+ Seq(attrDecimal -> ColumnStat(distinctCount = 3, min = Some(decMin),
max = Some(dec_0_60),
+ nullCount = 0, avgLen = 8, maxLen = 8)),
+ expectedRowCount = 3)
}
test("cdouble < 3.0") {
validateEstimatedStats(
- arDouble,
- Filter(LessThan(arDouble, Literal(3.0)),
childStatsTestPlan(Seq(arDouble), 10L)),
- ColumnStat(distinctCount = 2, min = Some(1.0), max = Some(3.0),
- nullCount = 0, avgLen = 8, maxLen = 8),
- 3)
+ Filter(LessThan(attrDouble, Literal(3.0)),
childStatsTestPlan(Seq(attrDouble), 10L)),
+ Seq(attrDouble -> ColumnStat(distinctCount = 2, min = Some(1.0), max
= Some(3.0),
+ nullCount = 0, avgLen = 8, maxLen = 8)),
+ expectedRowCount = 3)
}
test("cstring = 'A2'") {
validateEstimatedStats(
- arString,
- Filter(EqualTo(arString, Literal("A2")),
childStatsTestPlan(Seq(arString), 10L)),
- ColumnStat(distinctCount = 1, min = None, max = None,
- nullCount = 0, avgLen = 2, maxLen = 2),
- 1)
+ Filter(EqualTo(attrString, Literal("A2")),
childStatsTestPlan(Seq(attrString), 10L)),
+ Seq(attrString -> ColumnStat(distinctCount = 1, min = None, max =
None,
+ nullCount = 0, avgLen = 2, maxLen = 2)),
+ expectedRowCount = 1)
}
- // There is no min/max statistics for String type. We estimate 10 rows
returned.
- test("cstring < 'A2'") {
+ test("cstring < 'A2' - unsupported condition") {
validateEstimatedStats(
- arString,
- Filter(LessThan(arString, Literal("A2")),
childStatsTestPlan(Seq(arString), 10L)),
- ColumnStat(distinctCount = 10, min = None, max = None,
- nullCount = 0, avgLen = 2, maxLen = 2),
- 10)
+ Filter(LessThan(attrString, Literal("A2")),
childStatsTestPlan(Seq(attrString), 10L)),
+ Seq(attrString -> ColumnStat(distinctCount = 10, min = None, max =
None,
+ nullCount = 0, avgLen = 2, maxLen = 2)),
+ expectedRowCount = 10)
}
- // This is a corner test case. We want to test if we can handle the
case when the number of
- // valid values in IN clause is greater than the number of distinct
values for a given column.
- // For example, column has only 2 distinct values 1 and 6.
- // The predicate is: column IN (1, 2, 3, 4, 5).
test("cint IN (1, 2, 3, 4, 5)") {
+ // This is a corner test case. We want to test if we can handle the
case when the number of
+ // valid values in IN clause is greater than the number of distinct
values for a given column.
+ // For example, column has only 2 distinct values 1 and 6.
+ // The predicate is: column IN (1, 2, 3, 4, 5).
val cornerChildColStatInt = ColumnStat(distinctCount = 2, min =
Some(1), max = Some(6),
nullCount = 0, avgLen = 4, maxLen = 4)
val cornerChildStatsTestplan = StatsTestPlan(
- outputList = Seq(arInt),
+ outputList = Seq(attrInt),
rowCount = 2L,
- attributeStats = AttributeMap(Seq(arInt -> cornerChildColStatInt))
+ attributeStats = AttributeMap(Seq(attrInt -> cornerChildColStatInt))
)
validateEstimatedStats(
- arInt,
- Filter(InSet(arInt, Set(1, 2, 3, 4, 5)), cornerChildStatsTestplan),
- ColumnStat(distinctCount = 2, min = Some(1), max = Some(5),
- nullCount = 0, avgLen = 4, maxLen = 4),
- 2)
+ Filter(InSet(attrInt, Set(1, 2, 3, 4, 5)), cornerChildStatsTestplan),
+ Seq(attrInt -> ColumnStat(distinctCount = 2, min = Some(1), max =
Some(5),
+ nullCount = 0, avgLen = 4, maxLen = 4)),
+ expectedRowCount = 2)
}
private def childStatsTestPlan(outList: Seq[Attribute], tableRowCount:
BigInt): StatsTestPlan = {
StatsTestPlan(
outputList = outList,
rowCount = tableRowCount,
- attributeStats = AttributeMap(Seq(
- arInt -> childColStatInt,
- arBool -> childColStatBool,
- arDate -> childColStatDate,
- arDecimal -> childColStatDecimal,
- arDouble -> childColStatDouble,
- arString -> childColStatString
- ))
- )
+ attributeStats = AttributeMap(outList.map(a => a ->
attributeMap(a))))
}
private def validateEstimatedStats(
- ar: AttributeReference,
filterNode: Filter,
- expectedColStats: ColumnStat,
- rowCount: Int): Unit = {
-
- val expectedAttrStats = toAttributeMap(Seq(ar.name ->
expectedColStats), filterNode)
- val expectedSizeInBytes = getOutputSize(filterNode.output, rowCount,
expectedAttrStats)
-
- val filteredStats = filterNode.stats(conf)
- assert(filteredStats.sizeInBytes == expectedSizeInBytes)
- assert(filteredStats.rowCount.get == rowCount)
- assert(filteredStats.attributeStats(ar) == expectedColStats)
-
- // If the filter has a binary operator (including those nested inside
- // AND/OR/NOT), swap the sides of the attribte and the literal,
reverse the
- // operator, and then check again.
- val rewrittenFilter = filterNode transformExpressionsDown {
- case EqualTo(ar: AttributeReference, l: Literal) =>
- EqualTo(l, ar)
-
- case LessThan(ar: AttributeReference, l: Literal) =>
- GreaterThan(l, ar)
- case LessThanOrEqual(ar: AttributeReference, l: Literal) =>
- GreaterThanOrEqual(l, ar)
-
- case GreaterThan(ar: AttributeReference, l: Literal) =>
- LessThan(l, ar)
- case GreaterThanOrEqual(ar: AttributeReference, l: Literal) =>
- LessThanOrEqual(l, ar)
+ expectedColStats: Seq[(Attribute, ColumnStat)],
+ expectedRowCount: Int): Unit = {
+
+ // If the filter has a binary operator (including those nested inside
AND/OR/NOT), swap the
+ // sides of the attribute and the literal, reverse the operator, and
then check again.
+ val swappedFilter = filterNode transformExpressionsDown {
+ case EqualTo(attr: Attribute, l: Literal) =>
+ EqualTo(l, attr)
+
+ case LessThan(attr: Attribute, l: Literal) =>
+ GreaterThan(l, attr)
+ case LessThanOrEqual(attr: Attribute, l: Literal) =>
+ GreaterThanOrEqual(l, attr)
+
+ case GreaterThan(attr: Attribute, l: Literal) =>
+ LessThan(l, attr)
+ case GreaterThanOrEqual(attr: Attribute, l: Literal) =>
+ LessThanOrEqual(l, attr)
+ }
+
+ val testFilters = if (swappedFilter != filterNode) {
+ Seq(swappedFilter, filterNode)
--- End diff --
This is a good rewrite for method validateEstimatedStats.The current code
has better readability than tail recursion.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
