This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new ff8b94e [SPARK-34711][SQL][TESTS] Exercise code-gen enable/disable
code paths for SHJ in join test suites
ff8b94e is described below
commit ff8b94e6655a71736870508d980a7d924b3149a7
Author: Cheng Su <[email protected]>
AuthorDate: Wed Mar 10 23:34:09 2021 -0800
[SPARK-34711][SQL][TESTS] Exercise code-gen enable/disable code paths for
SHJ in join test suites
### What changes were proposed in this pull request?
Per comment in
https://github.com/apache/spark/pull/31802#discussion_r592068440 , we would
like to exercise whole stage code-gen enabled and disabled code paths in join
unit test suites. This is for better test coverage of shuffled hash join.
### Why are the changes needed?
Better test coverage.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing and added unit tests here.
Closes #31806 from c21/test-minor.
Authored-by: Cheng Su <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 9aa8f06313f53747fc30e7e6719e31693d2cd3f0)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala | 2 +-
.../scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala | 4 ++--
.../scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala
index fcbc0da..13848e5 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala
@@ -103,7 +103,7 @@ class ExistenceJoinSuite extends SparkPlanTest with
SharedSparkSession {
ProjectExec(output, FilterExec(condition, join))
}
- test(s"$testName using ShuffledHashJoin") {
+ testWithWholeStageCodegenOnAndOff(s"$testName using ShuffledHashJoin") { _
=>
extractJoinParts().foreach { case (_, leftKeys, rightKeys,
boundCondition, _, _, _) =>
withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
checkAnswer2(leftRows, rightRows, (left: SparkPlan, right:
SparkPlan) =>
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala
index f476c15..cf05c6b 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala
@@ -153,7 +153,7 @@ class InnerJoinSuite extends SparkPlanTest with
SharedSparkSession {
}
}
- test(s"$testName using ShuffledHashJoin (build=left)") {
+ testWithWholeStageCodegenOnAndOff(s"$testName using ShuffledHashJoin
(build=left)") { _ =>
extractJoinParts().foreach { case (_, leftKeys, rightKeys,
boundCondition, _, _, _) =>
withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
checkAnswer2(leftRows, rightRows, (leftPlan: SparkPlan, rightPlan:
SparkPlan) =>
@@ -165,7 +165,7 @@ class InnerJoinSuite extends SparkPlanTest with
SharedSparkSession {
}
}
- test(s"$testName using ShuffledHashJoin (build=right)") {
+ testWithWholeStageCodegenOnAndOff(s"$testName using ShuffledHashJoin
(build=right)") { _ =>
extractJoinParts().foreach { case (_, leftKeys, rightKeys,
boundCondition, _, _, _) =>
withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
checkAnswer2(leftRows, rightRows, (leftPlan: SparkPlan, rightPlan:
SparkPlan) =>
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala
index 238d37a..150d40d 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala
@@ -104,7 +104,7 @@ class OuterJoinSuite extends SparkPlanTest with
SharedSparkSession {
ExtractEquiJoinKeys.unapply(join)
}
- test(s"$testName using ShuffledHashJoin") {
+ testWithWholeStageCodegenOnAndOff(s"$testName using ShuffledHashJoin") { _
=>
extractJoinParts().foreach { case (_, leftKeys, rightKeys,
boundCondition, _, _, _) =>
withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
val buildSide = if (joinType == LeftOuter) BuildRight else BuildLeft
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
