kiszk commented on a change in pull request #28195: [SPARK-31425][SQL][CORE]
UnsafeKVExternalSorter should also respect UnsafeAlignedOffset
URL: https://github.com/apache/spark/pull/28195#discussion_r407406951
##########
File path:
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
##########
@@ -1055,28 +1056,31 @@ class HashAggregationQueryWithControlledFallbackSuite
extends AggregationQuerySu
Seq("true", "false").foreach { enableTwoLevelMaps =>
withSQLConf(SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key ->
enableTwoLevelMaps) {
- (1 to 3).foreach { fallbackStartsAt =>
- withSQLConf("spark.sql.TungstenAggregate.testFallbackStartsAt" ->
- s"${(fallbackStartsAt - 1).toString},
${fallbackStartsAt.toString}") {
- // Create a new df to make sure its physical operator picks up
- // spark.sql.TungstenAggregate.testFallbackStartsAt.
- // todo: remove it?
- val newActual = Dataset.ofRows(spark, actual.logicalPlan)
-
- QueryTest.getErrorMessageInCheckAnswer(newActual, expectedAnswer)
match {
- case Some(errorMessage) =>
- val newErrorMessage =
- s"""
- |The following aggregation query failed when using
HashAggregate with
- |controlled fallback (it falls back to bytes to bytes map
once it has processed
- |${fallbackStartsAt - 1} input rows and to sort-based
aggregation once it has
- |processed $fallbackStartsAt input rows). The query is
${actual.queryExecution}
- |
- |$errorMessage
+ Seq(4, 8).foreach { uaoSize =>
+ UnsafeAlignedOffset.setUaoSize(uaoSize)
+ (1 to 3).foreach { fallbackStartsAt =>
+ withSQLConf("spark.sql.TungstenAggregate.testFallbackStartsAt" ->
+ s"${(fallbackStartsAt - 1).toString},
${fallbackStartsAt.toString}") {
+ // Create a new df to make sure its physical operator picks up
+ // spark.sql.TungstenAggregate.testFallbackStartsAt.
+ // todo: remove it?
+ val newActual = Dataset.ofRows(spark, actual.logicalPlan)
+
+ QueryTest.getErrorMessageInCheckAnswer(newActual,
expectedAnswer) match {
+ case Some(errorMessage) =>
+ val newErrorMessage =
+ s"""
+ |The following aggregation query failed when using
HashAggregate with
+ |controlled fallback (it falls back to bytes to bytes
map once it has
+ |processed ${fallbackStartsAt - 1} input rows and to
sort-based aggregation
+ |once it has processed $fallbackStartsAt input rows).
+ |The query is ${actual.queryExecution}
+ |$errorMessage
""".stripMargin
Review comment:
nit: indentation issue
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
