Ngone51 commented on a change in pull request #28195: [SPARK-31425][SQL][CORE]
UnsafeKVExternalSorter/VariableLengthRowBasedKeyValueBatch should also respect
UnsafeAlignedOffset
URL: https://github.com/apache/spark/pull/28195#discussion_r408534911
##########
File path:
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
##########
@@ -1055,28 +1056,31 @@ class HashAggregationQueryWithControlledFallbackSuite
extends AggregationQuerySu
Seq("true", "false").foreach { enableTwoLevelMaps =>
withSQLConf(SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key ->
enableTwoLevelMaps) {
- (1 to 3).foreach { fallbackStartsAt =>
- withSQLConf("spark.sql.TungstenAggregate.testFallbackStartsAt" ->
- s"${(fallbackStartsAt - 1).toString},
${fallbackStartsAt.toString}") {
- // Create a new df to make sure its physical operator picks up
- // spark.sql.TungstenAggregate.testFallbackStartsAt.
- // todo: remove it?
- val newActual = Dataset.ofRows(spark, actual.logicalPlan)
-
- QueryTest.getErrorMessageInCheckAnswer(newActual, expectedAnswer)
match {
- case Some(errorMessage) =>
- val newErrorMessage =
- s"""
- |The following aggregation query failed when using
HashAggregate with
- |controlled fallback (it falls back to bytes to bytes map
once it has processed
- |${fallbackStartsAt - 1} input rows and to sort-based
aggregation once it has
- |processed $fallbackStartsAt input rows). The query is
${actual.queryExecution}
- |
- |$errorMessage
- """.stripMargin
-
- fail(newErrorMessage)
- case None => // Success
+ Seq(4, 8).foreach { uaoSize =>
+ UnsafeAlignedOffset.setUaoSize(uaoSize)
+ (1 to 3).foreach { fallbackStartsAt =>
+ withSQLConf("spark.sql.TungstenAggregate.testFallbackStartsAt" ->
+ s"${(fallbackStartsAt - 1).toString},
${fallbackStartsAt.toString}") {
+ // Create a new df to make sure its physical operator picks up
+ // spark.sql.TungstenAggregate.testFallbackStartsAt.
+ // todo: remove it?
+ val newActual = Dataset.ofRows(spark, actual.logicalPlan)
+
+ QueryTest.getErrorMessageInCheckAnswer(newActual,
expectedAnswer) match {
+ case Some(errorMessage) =>
+ val newErrorMessage =
+ s"""
+ |The following aggregation query failed when using
HashAggregate with
+ |controlled fallback (it falls back to bytes to bytes
map once it has
+ |processed ${fallbackStartsAt - 1} input rows and to
sort-based aggregation
+ |once it has processed $fallbackStartsAt input rows).
+ |The query is ${actual.queryExecution}
+ |$errorMessage
+ """.stripMargin
+
+ fail(newErrorMessage)
+ case None => // Success
+ }
Review comment:
Oh, yeah. I think we have to reset it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
