This is an automated email from the ASF dual-hosted git repository.
rui pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 588faae35 [VL] Enable SortShuffleSuite with ColumnarShuffleManager
(#5816)
588faae35 is described below
commit 588faae351bff29d868336f530aa72eb99a57083
Author: Ankita Victor <[email protected]>
AuthorDate: Wed May 29 18:33:29 2024 +0530
[VL] Enable SortShuffleSuite with ColumnarShuffleManager (#5816)
---
.../utils/clickhouse/ClickHouseTestSettings.scala | 1 +
.../gluten/utils/velox/VeloxTestSettings.scala | 2 ++
.../org/apache/spark/GlutenSortShuffleSuite.scala | 24 ++++++++++++++++++++++
.../utils/clickhouse/ClickHouseTestSettings.scala | 1 +
.../gluten/utils/velox/VeloxTestSettings.scala | 2 ++
.../org/apache/spark/GlutenSortShuffleSuite.scala | 24 ++++++++++++++++++++++
.../utils/clickhouse/ClickHouseTestSettings.scala | 1 +
.../gluten/utils/velox/VeloxTestSettings.scala | 2 ++
.../org/apache/spark/GlutenSortShuffleSuite.scala | 24 ++++++++++++++++++++++
.../utils/clickhouse/ClickHouseTestSettings.scala | 1 +
.../gluten/utils/velox/VeloxTestSettings.scala | 2 ++
.../org/apache/spark/GlutenSortShuffleSuite.scala | 24 ++++++++++++++++++++++
12 files changed, 108 insertions(+)
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index afc427cd3..2c34baa63 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -68,6 +68,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
false // nativeDoValidate failed due to spark conf cleanup
case "GlutenDataSourceV2SQLSuite" =>
false // nativeDoValidate failed due to spark conf cleanup
+ case "GlutenSortShuffleSuite" => false
case _ => true
}
preCheck && super.shouldRun(suiteName, testName)
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index c78d8230e..664cd37d1 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -18,6 +18,7 @@ package org.apache.gluten.utils.velox
import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings}
+import org.apache.spark.GlutenSortShuffleSuite
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.expressions._
import
org.apache.spark.sql.connector.{GlutenDataSourceV2DataFrameSessionCatalogSuite,
GlutenDataSourceV2DataFrameSuite, GlutenDataSourceV2FunctionSuite,
GlutenDataSourceV2SQLSessionCatalogSuite, GlutenDataSourceV2SQLSuite,
GlutenDataSourceV2Suite, GlutenFileDataSourceV2FallBackSuite,
GlutenLocalScanSuite, GlutenSupportsCatalogOptionsSuite,
GlutenTableCapabilityCheckSuite, GlutenWriteDistributionAndOrderingSuite}
@@ -229,6 +230,7 @@ class VeloxTestSettings extends BackendTestSettings {
// Spark round UT for round(3.1415,3) is not correct.
.exclude("round/bround")
enableSuite[GlutenMathFunctionsSuite]
+ enableSuite[GlutenSortShuffleSuite]
enableSuite[GlutenSortOrderExpressionsSuite]
enableSuite[GlutenBitwiseExpressionsSuite]
enableSuite[GlutenStringExpressionsSuite]
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala
new file mode 100644
index 000000000..338d7992e
--- /dev/null
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark
+
+class GlutenSortShuffleSuite extends SortShuffleSuite {
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ conf.set("spark.shuffle.manager",
"org.apache.spark.shuffle.sort.ColumnarShuffleManager")
+ }
+}
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 85f3f94cc..bb782fde3 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -74,6 +74,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
false // nativeDoValidate failed due to spark conf cleanup
case "GlutenBloomFilterAggregateQuerySuite" =>
!bloomFilterCases.contains(testName)
+ case "GlutenSortShuffleSuite" => false
case _ => true
}
preCheck && super.shouldRun(suiteName, testName)
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 3b32cebca..57c197622 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -18,6 +18,7 @@ package org.apache.gluten.utils.velox
import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings}
+import org.apache.spark.GlutenSortShuffleSuite
import org.apache.spark.sql._
import
org.apache.spark.sql.catalyst.expressions.{GlutenAnsiCastSuiteWithAnsiModeOff,
GlutenAnsiCastSuiteWithAnsiModeOn, GlutenArithmeticExpressionSuite,
GlutenBitwiseExpressionsSuite, GlutenCastSuite, GlutenCastSuiteWithAnsiModeOn,
GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite,
GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite,
GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite,
GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite,
GlutenIntervalExpre [...]
import org.apache.spark.sql.connector._
@@ -159,6 +160,7 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("random")
.exclude("SPARK-9127 codegen with long seed")
enableSuite[GlutenRegexpExpressionsSuite]
+ enableSuite[GlutenSortShuffleSuite]
enableSuite[GlutenSortOrderExpressionsSuite]
enableSuite[GlutenStringExpressionsSuite]
enableSuite[VeloxAdaptiveQueryExecSuite]
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala
new file mode 100644
index 000000000..338d7992e
--- /dev/null
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark
+
+class GlutenSortShuffleSuite extends SortShuffleSuite {
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ conf.set("spark.shuffle.manager",
"org.apache.spark.shuffle.sort.ColumnarShuffleManager")
+ }
+}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 069d697bd..7a3877427 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -76,6 +76,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
false // nativeDoValidate failed due to spark conf cleanup
case "GlutenBloomFilterAggregateQuerySuite" =>
!bloomFilterCases.contains(testName)
+ case "GlutenSortShuffleSuite" => false
case _ => true
}
preCheck && super.shouldRun(suiteName, testName)
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 505417aeb..6126844ad 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -18,6 +18,7 @@ package org.apache.gluten.utils.velox
import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings}
+import org.apache.spark.GlutenSortShuffleSuite
import org.apache.spark.sql._
import
org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite,
GlutenBitwiseExpressionsSuite, GlutenCastSuite,
GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite,
GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite,
GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite,
GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite,
GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite,
GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, Glu [...]
import
org.apache.spark.sql.connector.{GlutenDataSourceV2DataFrameSessionCatalogSuite,
GlutenDataSourceV2DataFrameSuite, GlutenDataSourceV2FunctionSuite,
GlutenDataSourceV2SQLSessionCatalogSuite, GlutenDataSourceV2SQLSuiteV1Filter,
GlutenDataSourceV2SQLSuiteV2Filter, GlutenDataSourceV2Suite,
GlutenDeleteFromTableSuite, GlutenDeltaBasedDeleteFromTableSuite,
GlutenFileDataSourceV2FallBackSuite, GlutenGroupBasedDeleteFromTableSuite,
GlutenKeyGroupedPartitioningSuite, GlutenLocalScanSuite, G [...]
@@ -137,6 +138,7 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("random")
.exclude("SPARK-9127 codegen with long seed")
enableSuite[GlutenRegexpExpressionsSuite]
+ enableSuite[GlutenSortShuffleSuite]
enableSuite[GlutenSortOrderExpressionsSuite]
enableSuite[GlutenStringExpressionsSuite]
enableSuite[VeloxAdaptiveQueryExecSuite]
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala
new file mode 100644
index 000000000..338d7992e
--- /dev/null
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark
+
+class GlutenSortShuffleSuite extends SortShuffleSuite {
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ conf.set("spark.shuffle.manager",
"org.apache.spark.shuffle.sort.ColumnarShuffleManager")
+ }
+}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 069d697bd..7a3877427 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -76,6 +76,7 @@ class ClickHouseTestSettings extends BackendTestSettings {
false // nativeDoValidate failed due to spark conf cleanup
case "GlutenBloomFilterAggregateQuerySuite" =>
!bloomFilterCases.contains(testName)
+ case "GlutenSortShuffleSuite" => false
case _ => true
}
preCheck && super.shouldRun(suiteName, testName)
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 27557f920..28f334878 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -18,6 +18,7 @@ package org.apache.gluten.utils.velox
import org.apache.gluten.utils.{BackendTestSettings, SQLQueryTestSettings}
+import org.apache.spark.GlutenSortShuffleSuite
import org.apache.spark.sql._
import
org.apache.spark.sql.catalyst.expressions.{GlutenArithmeticExpressionSuite,
GlutenBitwiseExpressionsSuite, GlutenCastSuite,
GlutenCollectionExpressionsSuite, GlutenComplexTypeSuite,
GlutenConditionalExpressionSuite, GlutenDateExpressionsSuite,
GlutenDecimalExpressionSuite, GlutenDecimalPrecisionSuite,
GlutenHashExpressionsSuite, GlutenHigherOrderFunctionsSuite,
GlutenIntervalExpressionsSuite, GlutenLiteralExpressionSuite,
GlutenMathExpressionsSuite, GlutenMiscExpressionsSuite, Glu [...]
import org.apache.spark.sql.connector._
@@ -140,6 +141,7 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("random")
.exclude("SPARK-9127 codegen with long seed")
enableSuite[GlutenRegexpExpressionsSuite]
+ enableSuite[GlutenSortShuffleSuite]
enableSuite[GlutenSortOrderExpressionsSuite]
enableSuite[GlutenStringExpressionsSuite]
enableSuite[VeloxAdaptiveQueryExecSuite]
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala
new file mode 100644
index 000000000..338d7992e
--- /dev/null
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark
+
+class GlutenSortShuffleSuite extends SortShuffleSuite {
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ conf.set("spark.shuffle.manager",
"org.apache.spark.shuffle.sort.ColumnarShuffleManager")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
