huaxingao commented on code in PR #41614:
URL: https://github.com/apache/spark/pull/41614#discussion_r1231740900
##########
sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala:
##########
@@ -1313,78 +1313,82 @@ class JoinSuite extends QueryTest with
SharedSparkSession with AdaptiveSparkPlan
test("SPARK-36612: Support left outer join build left or right outer join
build right in " +
"shuffled hash join") {
- val inputDFs = Seq(
- // Test unique join key
- (spark.range(10).selectExpr("id as k1"),
- spark.range(30).selectExpr("id as k2"),
- $"k1" === $"k2"),
- // Test non-unique join key
- (spark.range(10).selectExpr("id % 5 as k1"),
- spark.range(30).selectExpr("id % 5 as k2"),
- $"k1" === $"k2"),
- // Test empty build side
- (spark.range(10).selectExpr("id as k1").filter("k1 < -1"),
- spark.range(30).selectExpr("id as k2"),
- $"k1" === $"k2"),
- // Test empty stream side
- (spark.range(10).selectExpr("id as k1"),
- spark.range(30).selectExpr("id as k2").filter("k2 < -1"),
- $"k1" === $"k2"),
- // Test empty build and stream side
- (spark.range(10).selectExpr("id as k1").filter("k1 < -1"),
- spark.range(30).selectExpr("id as k2").filter("k2 < -1"),
- $"k1" === $"k2"),
- // Test string join key
- (spark.range(10).selectExpr("cast(id * 3 as string) as k1"),
- spark.range(30).selectExpr("cast(id as string) as k2"),
- $"k1" === $"k2"),
- // Test build side at right
- (spark.range(30).selectExpr("cast(id / 3 as string) as k1"),
- spark.range(10).selectExpr("cast(id as string) as k2"),
- $"k1" === $"k2"),
- // Test NULL join key
- (spark.range(10).map(i => if (i % 2 == 0) i else null).selectExpr("value
as k1"),
- spark.range(30).map(i => if (i % 4 == 0) i else
null).selectExpr("value as k2"),
- $"k1" === $"k2"),
- (spark.range(10).map(i => if (i % 3 == 0) i else null).selectExpr("value
as k1"),
- spark.range(30).map(i => if (i % 5 == 0) i else
null).selectExpr("value as k2"),
- $"k1" === $"k2"),
- // Test multiple join keys
- (spark.range(10).map(i => if (i % 2 == 0) i else null).selectExpr(
- "value as k1", "cast(value % 5 as short) as k2", "cast(value * 3 as
long) as k3"),
- spark.range(30).map(i => if (i % 3 == 0) i else null).selectExpr(
- "value as k4", "cast(value % 5 as short) as k5", "cast(value * 3 as
long) as k6"),
- $"k1" === $"k4" && $"k2" === $"k5" && $"k3" === $"k6")
- )
+ withSQLConf(SQLConf.ENABLE_BUILD_SIDE_OUTER_SHUFFLED_HASH_JOIN_CODEGEN.key
-> "false") {
Review Comment:
Shall we test both the codegen and non-codegen codepath? This test seems to
have more coverage than the test in `WholeStageCodegenSuite`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
