This is an automated email from the ASF dual-hosted git repository. changchen pushed a commit to branch fix/11550-system-property-trait in repository https://gitbox.apache.org/repos/asf/gluten.git
commit 31dbdbfa8ac59ed8cfbfc636dc5718cafcc0d320 Author: Chang chen <[email protected]> AuthorDate: Mon Mar 30 04:58:54 2026 +0000 [GLUTEN-11550][UT] Add GlutenTestSetWithSystemPropertyTrait and enable 6 suites Add a reusable trait for test suites whose parent creates per-test SparkContext/SparkSession (via LocalSparkContext/LocalSparkSession). It injects GlutenPlugin config via system properties so per-test sessions inherit it, avoiding shared session conflicts. Enable 6 suites by switching to this trait: - GlutenSQLExecutionSuite (SQLExecutionSuite: new SparkContext internally) - GlutenSQLJsonProtocolSuite (SQLJsonProtocolSuite with LocalSparkSession) - GlutenShufflePartitionsUtilSuite (ShufflePartitionsUtilSuite with LocalSparkContext) - GlutenExternalAppendOnlyUnsafeRowArraySuite (with LocalSparkContext) - GlutenUnsafeRowSerializerSuite (UnsafeRowSerializerSuite with LocalSparkSession) - GlutenSparkSessionJobTaggingAndCancellationSuite (with LocalSparkContext) Co-authored-by: Copilot <[email protected]> --- .../sql/GlutenTestSetWithSystemPropertyTrait.scala | 48 ++++++++++++++++++++++ .../gluten/utils/velox/VeloxTestSettings.scala | 10 ++--- ...parkSessionJobTaggingAndCancellationSuite.scala | 2 +- ...utenExternalAppendOnlyUnsafeRowArraySuite.scala | 4 +- .../sql/execution/GlutenSQLExecutionSuite.scala | 4 +- .../sql/execution/GlutenSQLJsonProtocolSuite.scala | 6 ++- .../GlutenShufflePartitionsUtilSuite.scala | 6 ++- .../execution/GlutenUnsafeRowSerializerSuite.scala | 6 ++- .../gluten/utils/velox/VeloxTestSettings.scala | 10 ++--- ...parkSessionJobTaggingAndCancellationSuite.scala | 4 +- ...utenExternalAppendOnlyUnsafeRowArraySuite.scala | 4 +- .../sql/execution/GlutenSQLExecutionSuite.scala | 4 +- .../sql/execution/GlutenSQLJsonProtocolSuite.scala | 6 ++- .../GlutenShufflePartitionsUtilSuite.scala | 6 ++- .../execution/GlutenUnsafeRowSerializerSuite.scala | 6 ++- 15 files changed, 92 insertions(+), 34 deletions(-) diff --git a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestSetWithSystemPropertyTrait.scala b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestSetWithSystemPropertyTrait.scala new file mode 100644 index 0000000000..0b38f1c016 --- /dev/null +++ b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestSetWithSystemPropertyTrait.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +/** + * A test trait for suites whose parent creates per-test SparkContext/SparkSession (via + * LocalSparkContext, LocalSparkSession, or direct `new SparkContext`). + * + * Injects GlutenPlugin config via system properties so per-test sessions inherit it. SparkConf + * reads system properties prefixed with "spark." as defaults during construction. + * + * Do NOT use GlutenTestsTrait or GlutenSQLTestsTrait for these suites -- they create a shared + * SparkSession in beforeAll() that conflicts with per-test SparkContext creation. + */ +trait GlutenTestSetWithSystemPropertyTrait extends GlutenTestsCommonTrait { + + override def beforeAll(): Unit = { + System.setProperty("spark.plugins", "org.apache.gluten.GlutenPlugin") + System.setProperty("spark.memory.offHeap.enabled", "true") + System.setProperty("spark.memory.offHeap.size", "1024MB") + System.setProperty( + "spark.shuffle.manager", + "org.apache.spark.shuffle.sort.ColumnarShuffleManager") + super.beforeAll() + } + + override def afterAll(): Unit = { + super.afterAll() + System.clearProperty("spark.plugins") + System.clearProperty("spark.memory.offHeap.enabled") + System.clearProperty("spark.memory.offHeap.size") + System.clearProperty("spark.shuffle.manager") + } +} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 80a33021d1..81adb9f958 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -680,7 +680,7 @@ class VeloxTestSettings extends BackendTestSettings { // TODO: 4.x enableSuite[GlutenDataSourceScanExecRedactionSuite] // 2 failures // TODO: 4.x enableSuite[GlutenDataSourceV2ScanExecRedactionSuite] // 2 failures enableSuite[GlutenExecuteImmediateEndToEndSuite] - // TODO: 4.x enableSuite[GlutenExternalAppendOnlyUnsafeRowArraySuite] // 14 failures + enableSuite[GlutenExternalAppendOnlyUnsafeRowArraySuite] enableSuite[GlutenGlobalTempViewSuite] enableSuite[GlutenGlobalTempViewTestSuite] enableSuite[GlutenGroupedIteratorSuite] @@ -696,10 +696,10 @@ class VeloxTestSettings extends BackendTestSettings { // TODO: 4.x enableSuite[GlutenRemoveRedundantProjectsSuite] // 14 failures // TODO: 4.x enableSuite[GlutenRemoveRedundantSortsSuite] // 1 failure enableSuite[GlutenRowToColumnConverterSuite] - // TODO: 4.x enableSuite[GlutenSQLExecutionSuite] // 1 failure + enableSuite[GlutenSQLExecutionSuite] enableSuite[GlutenSQLFunctionSuite] - // TODO: 4.x enableSuite[GlutenSQLJsonProtocolSuite] // 1 failure - // TODO: 4.x enableSuite[GlutenShufflePartitionsUtilSuite] // 1 failure + enableSuite[GlutenSQLJsonProtocolSuite] + enableSuite[GlutenShufflePartitionsUtilSuite] // TODO: 4.x enableSuite[GlutenSimpleSQLViewSuite] // 1 failure // TODO: 4.x enableSuite[GlutenSparkPlanSuite] // 1 failure enableSuite[GlutenSparkPlannerSuite] @@ -707,7 +707,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSparkSqlParserSuite] enableSuite[GlutenUnsafeFixedWidthAggregationMapSuite] enableSuite[GlutenUnsafeKVExternalSorterSuite] - // TODO: 4.x enableSuite[GlutenUnsafeRowSerializerSuite] // 1 failure + enableSuite[GlutenUnsafeRowSerializerSuite] // TODO: 4.x enableSuite[GlutenWholeStageCodegenSparkSubmitSuite] // 1 failure // TODO: 4.x enableSuite[GlutenWholeStageCodegenSuite] // 24 failures enableSuite[GlutenBroadcastExchangeSuite] diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenSparkSessionJobTaggingAndCancellationSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenSparkSessionJobTaggingAndCancellationSuite.scala index 8fbb53e24c..d24641fb1f 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenSparkSessionJobTaggingAndCancellationSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenSparkSessionJobTaggingAndCancellationSuite.scala @@ -18,4 +18,4 @@ package org.apache.spark.sql class GlutenSparkSessionJobTaggingAndCancellationSuite extends SparkSessionJobTaggingAndCancellationSuite - with GlutenTestsTrait {} + with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenExternalAppendOnlyUnsafeRowArraySuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenExternalAppendOnlyUnsafeRowArraySuite.scala index f947c4f406..4229733100 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenExternalAppendOnlyUnsafeRowArraySuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenExternalAppendOnlyUnsafeRowArraySuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.spark.sql.GlutenTestsTrait +import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait class GlutenExternalAppendOnlyUnsafeRowArraySuite extends ExternalAppendOnlyUnsafeRowArraySuite - with GlutenTestsTrait {} + with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSQLExecutionSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSQLExecutionSuite.scala index 51c06b3b69..05b1d5fea1 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSQLExecutionSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSQLExecutionSuite.scala @@ -16,6 +16,6 @@ */ package org.apache.spark.sql.execution -import org.apache.spark.sql.GlutenTestsTrait +import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -class GlutenSQLExecutionSuite extends SQLExecutionSuite with GlutenTestsTrait {} +class GlutenSQLExecutionSuite extends SQLExecutionSuite with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSQLJsonProtocolSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSQLJsonProtocolSuite.scala index 7ce28aae89..12af785acd 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSQLJsonProtocolSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenSQLJsonProtocolSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.spark.sql.GlutenTestsTrait +import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -class GlutenSQLJsonProtocolSuite extends SQLJsonProtocolSuite with GlutenTestsTrait {} +class GlutenSQLJsonProtocolSuite + extends SQLJsonProtocolSuite + with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenShufflePartitionsUtilSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenShufflePartitionsUtilSuite.scala index 866d6cea54..0184af6774 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenShufflePartitionsUtilSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenShufflePartitionsUtilSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.spark.sql.GlutenTestsTrait +import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -class GlutenShufflePartitionsUtilSuite extends ShufflePartitionsUtilSuite with GlutenTestsTrait {} +class GlutenShufflePartitionsUtilSuite + extends ShufflePartitionsUtilSuite + with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenUnsafeRowSerializerSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenUnsafeRowSerializerSuite.scala index 39a52a35dd..0f76b5fc2a 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenUnsafeRowSerializerSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/execution/GlutenUnsafeRowSerializerSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.spark.sql.GlutenTestsTrait +import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -class GlutenUnsafeRowSerializerSuite extends UnsafeRowSerializerSuite with GlutenTestsTrait {} +class GlutenUnsafeRowSerializerSuite + extends UnsafeRowSerializerSuite + with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index be6e89bbd3..cca8acfdb0 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -663,7 +663,7 @@ class VeloxTestSettings extends BackendTestSettings { // TODO: 4.x enableSuite[GlutenDataSourceScanExecRedactionSuite] // 2 failures // TODO: 4.x enableSuite[GlutenDataSourceV2ScanExecRedactionSuite] // 2 failures enableSuite[GlutenExecuteImmediateEndToEndSuite] - // TODO: 4.x enableSuite[GlutenExternalAppendOnlyUnsafeRowArraySuite] // 14 failures + enableSuite[GlutenExternalAppendOnlyUnsafeRowArraySuite] enableSuite[GlutenGlobalTempViewSuite] enableSuite[GlutenGlobalTempViewTestSuite] enableSuite[GlutenGroupedIteratorSuite] @@ -679,10 +679,10 @@ class VeloxTestSettings extends BackendTestSettings { // TODO: 4.x enableSuite[GlutenRemoveRedundantProjectsSuite] // 14 failures // TODO: 4.x enableSuite[GlutenRemoveRedundantSortsSuite] // 1 failure enableSuite[GlutenRowToColumnConverterSuite] - // TODO: 4.x enableSuite[GlutenSQLExecutionSuite] // 1 failure + enableSuite[GlutenSQLExecutionSuite] enableSuite[GlutenSQLFunctionSuite] - // TODO: 4.x enableSuite[GlutenSQLJsonProtocolSuite] // 1 failure - // TODO: 4.x enableSuite[GlutenShufflePartitionsUtilSuite] // 1 failure + enableSuite[GlutenSQLJsonProtocolSuite] + enableSuite[GlutenShufflePartitionsUtilSuite] // TODO: 4.x enableSuite[GlutenSimpleSQLViewSuite] // 2 failures // TODO: 4.x enableSuite[GlutenSparkPlanSuite] // 1 failure enableSuite[GlutenSparkPlannerSuite] @@ -690,7 +690,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenSparkSqlParserSuite] enableSuite[GlutenUnsafeFixedWidthAggregationMapSuite] enableSuite[GlutenUnsafeKVExternalSorterSuite] - // TODO: 4.x enableSuite[GlutenUnsafeRowSerializerSuite] // 1 failure + enableSuite[GlutenUnsafeRowSerializerSuite] // TODO: 4.x enableSuite[GlutenWholeStageCodegenSparkSubmitSuite] // 1 failure // TODO: 4.x enableSuite[GlutenWholeStageCodegenSuite] // 24 failures enableSuite[GlutenBroadcastExchangeSuite] diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenSparkSessionJobTaggingAndCancellationSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenSparkSessionJobTaggingAndCancellationSuite.scala index 6befbbe213..d24641fb1f 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenSparkSessionJobTaggingAndCancellationSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenSparkSessionJobTaggingAndCancellationSuite.scala @@ -16,8 +16,6 @@ */ package org.apache.spark.sql -import org.apache.spark.sql.shim.GlutenTestsTrait - class GlutenSparkSessionJobTaggingAndCancellationSuite extends SparkSessionJobTaggingAndCancellationSuite - with GlutenTestsTrait {} + with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenExternalAppendOnlyUnsafeRowArraySuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenExternalAppendOnlyUnsafeRowArraySuite.scala index cd3aeb6438..4229733100 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenExternalAppendOnlyUnsafeRowArraySuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenExternalAppendOnlyUnsafeRowArraySuite.scala @@ -16,8 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.spark.sql.shim.GlutenTestsTrait +import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait class GlutenExternalAppendOnlyUnsafeRowArraySuite extends ExternalAppendOnlyUnsafeRowArraySuite - with GlutenTestsTrait {} + with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSQLExecutionSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSQLExecutionSuite.scala index 27865af762..05b1d5fea1 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSQLExecutionSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSQLExecutionSuite.scala @@ -16,6 +16,6 @@ */ package org.apache.spark.sql.execution -import org.apache.spark.sql.shim.GlutenTestsTrait +import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -class GlutenSQLExecutionSuite extends SQLExecutionSuite with GlutenTestsTrait {} +class GlutenSQLExecutionSuite extends SQLExecutionSuite with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSQLJsonProtocolSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSQLJsonProtocolSuite.scala index 2ca7b4380d..12af785acd 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSQLJsonProtocolSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenSQLJsonProtocolSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.spark.sql.shim.GlutenTestsTrait +import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -class GlutenSQLJsonProtocolSuite extends SQLJsonProtocolSuite with GlutenTestsTrait {} +class GlutenSQLJsonProtocolSuite + extends SQLJsonProtocolSuite + with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenShufflePartitionsUtilSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenShufflePartitionsUtilSuite.scala index 5a6595bd9e..0184af6774 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenShufflePartitionsUtilSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenShufflePartitionsUtilSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.spark.sql.shim.GlutenTestsTrait +import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -class GlutenShufflePartitionsUtilSuite extends ShufflePartitionsUtilSuite with GlutenTestsTrait {} +class GlutenShufflePartitionsUtilSuite + extends ShufflePartitionsUtilSuite + with GlutenTestSetWithSystemPropertyTrait {} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenUnsafeRowSerializerSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenUnsafeRowSerializerSuite.scala index d81f8d58e7..0f76b5fc2a 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenUnsafeRowSerializerSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/execution/GlutenUnsafeRowSerializerSuite.scala @@ -16,6 +16,8 @@ */ package org.apache.spark.sql.execution -import org.apache.spark.sql.shim.GlutenTestsTrait +import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait -class GlutenUnsafeRowSerializerSuite extends UnsafeRowSerializerSuite with GlutenTestsTrait {} +class GlutenUnsafeRowSerializerSuite + extends UnsafeRowSerializerSuite + with GlutenTestSetWithSystemPropertyTrait {} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
