This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push: new ff8b94e [SPARK-34711][SQL][TESTS] Exercise code-gen enable/disable code paths for SHJ in join test suites ff8b94e is described below commit ff8b94e6655a71736870508d980a7d924b3149a7 Author: Cheng Su <chen...@fb.com> AuthorDate: Wed Mar 10 23:34:09 2021 -0800 [SPARK-34711][SQL][TESTS] Exercise code-gen enable/disable code paths for SHJ in join test suites ### What changes were proposed in this pull request? Per comment in https://github.com/apache/spark/pull/31802#discussion_r592068440 , we would like to exercise whole stage code-gen enabled and disabled code paths in join unit test suites. This is for better test coverage of shuffled hash join. ### Why are the changes needed? Better test coverage. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing and added unit tests here. Closes #31806 from c21/test-minor. Authored-by: Cheng Su <chen...@fb.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> (cherry picked from commit 9aa8f06313f53747fc30e7e6719e31693d2cd3f0) Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala | 2 +- .../scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala | 4 ++-- .../scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala index fcbc0da..13848e5 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/ExistenceJoinSuite.scala @@ -103,7 +103,7 @@ class ExistenceJoinSuite extends SparkPlanTest with SharedSparkSession { ProjectExec(output, FilterExec(condition, join)) } - test(s"$testName using ShuffledHashJoin") { + testWithWholeStageCodegenOnAndOff(s"$testName using ShuffledHashJoin") { _ => extractJoinParts().foreach { case (_, leftKeys, rightKeys, boundCondition, _, _, _) => withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") { checkAnswer2(leftRows, rightRows, (left: SparkPlan, right: SparkPlan) => diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala index f476c15..cf05c6b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/InnerJoinSuite.scala @@ -153,7 +153,7 @@ class InnerJoinSuite extends SparkPlanTest with SharedSparkSession { } } - test(s"$testName using ShuffledHashJoin (build=left)") { + testWithWholeStageCodegenOnAndOff(s"$testName using ShuffledHashJoin (build=left)") { _ => extractJoinParts().foreach { case (_, leftKeys, rightKeys, boundCondition, _, _, _) => withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") { checkAnswer2(leftRows, rightRows, (leftPlan: SparkPlan, rightPlan: SparkPlan) => @@ -165,7 +165,7 @@ class InnerJoinSuite extends SparkPlanTest with SharedSparkSession { } } - test(s"$testName using ShuffledHashJoin (build=right)") { + testWithWholeStageCodegenOnAndOff(s"$testName using ShuffledHashJoin (build=right)") { _ => extractJoinParts().foreach { case (_, leftKeys, rightKeys, boundCondition, _, _, _) => withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") { checkAnswer2(leftRows, rightRows, (leftPlan: SparkPlan, rightPlan: SparkPlan) => diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala index 238d37a..150d40d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/OuterJoinSuite.scala @@ -104,7 +104,7 @@ class OuterJoinSuite extends SparkPlanTest with SharedSparkSession { ExtractEquiJoinKeys.unapply(join) } - test(s"$testName using ShuffledHashJoin") { + testWithWholeStageCodegenOnAndOff(s"$testName using ShuffledHashJoin") { _ => extractJoinParts().foreach { case (_, leftKeys, rightKeys, boundCondition, _, _, _) => withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") { val buildSide = if (joinType == LeftOuter) BuildRight else BuildLeft --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org