This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new da5259566 [VL] Add tests for Velox SMJ's coverage (#7195)
da5259566 is described below
commit da525956635d3c06d3a715a27b4b199912f23d84
Author: Hongze Zhang <[email protected]>
AuthorDate: Thu Sep 12 09:24:35 2024 +0800
[VL] Add tests for Velox SMJ's coverage (#7195)
---
.../utils/clickhouse/ClickHouseTestSettings.scala | 6 +++---
.../apache/gluten/utils/velox/VeloxTestSettings.scala | 13 ++++++++++---
.../sql/execution/joins/GlutenInnerJoinSuite.scala | 17 ++++++++++++++++-
.../sql/execution/joins/GlutenOuterJoinSuite.scala | 17 ++++++++++++++++-
4 files changed, 45 insertions(+), 8 deletions(-)
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 7e73ae9c1..77d4cb1ce 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.execution.datasources.parquet._
import org.apache.spark.sql.execution.datasources.text.{GlutenTextV1Suite,
GlutenTextV2Suite}
import
org.apache.spark.sql.execution.datasources.v2.{GlutenDataSourceV2StrategySuite,
GlutenFileTableSuite, GlutenV2PredicateSuite}
import org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
-import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite,
GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite}
+import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite,
GlutenExistenceJoinSuite, GlutenInnerJoinSuiteForceShjOn,
GlutenOuterJoinSuiteForceShjOn}
import org.apache.spark.sql.extension.{GlutenCustomerExtensionSuite,
GlutenSessionExtensionSuite}
import org.apache.spark.sql.gluten.GlutenFallbackSuite
import org.apache.spark.sql.hive.execution.GlutenHiveSQLQueryCHSuite
@@ -1624,7 +1624,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("test composed unique condition (both non-equal) for left anti
join using ShuffledHashJoin (whole-stage-codegen on)")
.exclude("test composed unique condition (both non-equal) for left anti
join using SortMergeJoin (whole-stage-codegen off)")
.exclude("test composed unique condition (both non-equal) for left anti
join using SortMergeJoin (whole-stage-codegen on)")
- enableSuite[GlutenInnerJoinSuite]
+ enableSuite[GlutenInnerJoinSuiteForceShjOn]
.exclude(
"inner join, one match per row using ShuffledHashJoin (build=left)
(whole-stage-codegen off)")
.exclude(
@@ -1673,7 +1673,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("SPARK-15822 - test structs as keys using BroadcastNestedLoopJoin
build left (whole-stage-codegen on)")
.exclude("SPARK-15822 - test structs as keys using BroadcastNestedLoopJoin
build right (whole-stage-codegen off)")
.exclude("SPARK-15822 - test structs as keys using BroadcastNestedLoopJoin
build right (whole-stage-codegen on)")
- enableSuite[GlutenOuterJoinSuite]
+ enableSuite[GlutenOuterJoinSuiteForceShjOn]
.exclude("basic left outer join using ShuffledHashJoin
(whole-stage-codegen off)")
.exclude("basic left outer join using ShuffledHashJoin
(whole-stage-codegen on)")
.exclude("basic left outer join using SortMergeJoin (whole-stage-codegen
off)")
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index baabff3f7..e4706caa7 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -35,7 +35,7 @@ import org.apache.spark.sql.execution.datasources.parquet._
import org.apache.spark.sql.execution.datasources.text.{GlutenTextV1Suite,
GlutenTextV2Suite}
import
org.apache.spark.sql.execution.datasources.v2.{GlutenDataSourceV2StrategySuite,
GlutenFileTableSuite, GlutenV2PredicateSuite}
import org.apache.spark.sql.execution.exchange.GlutenEnsureRequirementsSuite
-import org.apache.spark.sql.execution.joins.{GlutenBroadcastJoinSuite,
GlutenExistenceJoinSuite, GlutenInnerJoinSuite, GlutenOuterJoinSuite}
+import org.apache.spark.sql.execution.joins._
import
org.apache.spark.sql.extension.{GlutenCollapseProjectExecTransformerSuite,
GlutenSessionExtensionSuite, TestFileSourceScanExecTransformer}
import org.apache.spark.sql.gluten.GlutenFallbackSuite
import org.apache.spark.sql.hive.execution.GlutenHiveSQLQuerySuite
@@ -879,8 +879,15 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("broadcast join where streamed side's output partitioning is
HashPartitioning")
enableSuite[GlutenExistenceJoinSuite]
- enableSuite[GlutenInnerJoinSuite]
- enableSuite[GlutenOuterJoinSuite]
+ enableSuite[GlutenInnerJoinSuiteForceShjOn]
+ enableSuite[GlutenInnerJoinSuiteForceShjOff]
+ enableSuite[GlutenOuterJoinSuiteForceShjOn]
+ enableSuite[GlutenOuterJoinSuiteForceShjOff]
+ // Caused by Velox SMJ result mismatches with Spark.
+ .exclude("basic right outer join using SortMergeJoin (whole-stage-codegen
off)")
+ .exclude("basic right outer join using SortMergeJoin (whole-stage-codegen
on)")
+ .exclude("right outer join with unique keys using SortMergeJoin
(whole-stage-codegen off)")
+ .exclude("right outer join with unique keys using SortMergeJoin
(whole-stage-codegen on)")
enableSuite[FallbackStrategiesSuite]
enableSuite[GlutenBroadcastExchangeSuite]
enableSuite[GlutenLocalBroadcastExchangeSuite]
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala
index 745f550ae..d31936211 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenInnerJoinSuite.scala
@@ -16,6 +16,21 @@
*/
package org.apache.spark.sql.execution.joins
+import org.apache.gluten.GlutenConfig
+
+import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsBaseTrait
-class GlutenInnerJoinSuite extends InnerJoinSuite with GlutenSQLTestsBaseTrait
{}
+class GlutenInnerJoinSuiteForceShjOn extends InnerJoinSuite with
GlutenSQLTestsBaseTrait {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key, "true")
+ }
+}
+
+class GlutenInnerJoinSuiteForceShjOff extends InnerJoinSuite with
GlutenSQLTestsBaseTrait {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key, "false")
+ }
+}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala
index c915c7369..9c45eae15 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenOuterJoinSuite.scala
@@ -16,6 +16,21 @@
*/
package org.apache.spark.sql.execution.joins
+import org.apache.gluten.GlutenConfig
+
+import org.apache.spark.SparkConf
import org.apache.spark.sql.GlutenSQLTestsBaseTrait
-class GlutenOuterJoinSuite extends OuterJoinSuite with GlutenSQLTestsBaseTrait
{}
+class GlutenOuterJoinSuiteForceShjOn extends OuterJoinSuite with
GlutenSQLTestsBaseTrait {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key, "true")
+ }
+}
+
+class GlutenOuterJoinSuiteForceShjOff extends OuterJoinSuite with
GlutenSQLTestsBaseTrait {
+ override def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED.key, "false")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
