This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new fc04a630c [GLUTEN-6110] Parallel run gluten ut and spark ut (#6090)
fc04a630c is described below
commit fc04a630c9e291c8ea32ede81074ce3f6bb151e2
Author: Wenzheng Liu <[email protected]>
AuthorDate: Mon Jun 17 16:11:41 2024 +0800
[GLUTEN-6110] Parallel run gluten ut and spark ut (#6090)
1. Add ut notes
2. move page index test to gluten package
---
.../execution}/compatibility/GlutenFunctionSuite.scala | 4 ++--
.../execution}/parquet/GlutenParquetColumnIndexSuite.scala | 6 +++---
.../execution}/parquet/GlutenParquetFilterSuite.scala | 9 +++++----
.../{spark/sql => }/gluten/test/GlutenSQLTestUtils.scala | 2 +-
.../apache/{spark/sql => }/gluten/test/GlutenTPCBase.scala | 12 ++++++------
.../{spark/sql => }/gluten/test/GlutenTPCHBase.scala | 2 ++
docs/developers/NewToGluten.md | 14 ++++++++++++++
7 files changed, 33 insertions(+), 16 deletions(-)
diff --git
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenFunctionSuite.scala
similarity index 96%
rename from
backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala
rename to
backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenFunctionSuite.scala
index aaee82412..8a20558c4 100644
---
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/compatibility/GlutenFunctionSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenFunctionSuite.scala
@@ -14,13 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.sql.gluten.compatibility
+package org.apache.gluten.execution.compatibility
import org.apache.gluten.execution.GlutenClickHouseWholeStageTransformerSuite
+import org.apache.gluten.test.GlutenSQLTestUtils
import org.apache.gluten.utils.UTSystemParameters
import org.apache.spark.internal.Logging
-import org.apache.spark.sql.gluten.test.GlutenSQLTestUtils
class GlutenFunctionSuite
extends GlutenClickHouseWholeStageTransformerSuite
diff --git
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetColumnIndexSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/parquet/GlutenParquetColumnIndexSuite.scala
similarity index 95%
rename from
backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetColumnIndexSuite.scala
rename to
backends-clickhouse/src/test/scala/org/apache/gluten/execution/parquet/GlutenParquetColumnIndexSuite.scala
index 05ed7ed6b..0311594a1 100644
---
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetColumnIndexSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/parquet/GlutenParquetColumnIndexSuite.scala
@@ -14,15 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.sql.gluten.parquet
+package org.apache.gluten.execution.parquet
import org.apache.gluten.execution.{FileSourceScanExecTransformer,
GlutenClickHouseWholeStageTransformerSuite}
+import org.apache.gluten.test.GlutenSQLTestUtils
import org.apache.gluten.utils.UTSystemParameters
import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.gluten.test.GlutenSQLTestUtils
import org.apache.spark.sql.internal.SQLConf
case class ParquetData(
@@ -98,6 +98,6 @@ class GlutenParquetColumnIndexSuite
}
override protected def sparkConf: SparkConf =
super.sparkConf
- .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
+ .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
.set("spark.gluten.sql.columnar.backend.ch.runtime_config.use_local_format",
"true")
}
diff --git
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetFilterSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/parquet/GlutenParquetFilterSuite.scala
similarity index 98%
rename from
backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetFilterSuite.scala
rename to
backends-clickhouse/src/test/scala/org/apache/gluten/execution/parquet/GlutenParquetFilterSuite.scala
index 1022b17e4..a1b5801da 100644
---
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/parquet/GlutenParquetFilterSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/parquet/GlutenParquetFilterSuite.scala
@@ -14,15 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.sql.gluten.parquet
+package org.apache.gluten.execution.parquet
import org.apache.gluten.execution.{FileSourceScanExecTransformer,
GlutenClickHouseWholeStageTransformerSuite}
+import org.apache.gluten.test.GlutenSQLTestUtils
import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.gluten.test.{GlutenSQLTestUtils, GlutenTPCHBase}
+import org.apache.spark.sql.gluten.test.GlutenTPCHBase
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.Decimal
@@ -45,8 +46,8 @@ class GlutenParquetFilterSuite
override protected def sparkConf: SparkConf =
super.sparkConf
- .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
- .set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD, -1L) // disable broadcast
+ .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
+ .set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1") // disable broadcast
private val result: Array[Map[String, Seq[Predicate]]] = Array(
Map( // q1
diff --git
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenSQLTestUtils.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenSQLTestUtils.scala
similarity index 96%
rename from
backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenSQLTestUtils.scala
rename to
backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenSQLTestUtils.scala
index 09ffb91ff..9888baf9a 100644
---
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenSQLTestUtils.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenSQLTestUtils.scala
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.sql.gluten.test
+package org.apache.gluten.test
import org.apache.gluten.GlutenConfig
diff --git
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenTPCBase.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenTPCBase.scala
similarity index 80%
rename from
backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenTPCBase.scala
rename to
backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenTPCBase.scala
index 21c6abb56..224ad6443 100644
---
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenTPCBase.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenTPCBase.scala
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.spark.sql.gluten.test
+package org.apache.gluten.test
import org.apache.spark.SparkConf
import org.apache.spark.sql.internal.SQLConf
@@ -27,12 +27,12 @@ trait GlutenTPCBase extends SharedSparkSession {
override protected def sparkConf: SparkConf = {
if (injectStats) {
super.sparkConf
- .set(SQLConf.MAX_TO_STRING_FIELDS, Int.MaxValue)
- .set(SQLConf.CBO_ENABLED, true)
- .set(SQLConf.PLAN_STATS_ENABLED, true)
- .set(SQLConf.JOIN_REORDER_ENABLED, true)
+ .set(SQLConf.MAX_TO_STRING_FIELDS.key, s"${Int.MaxValue}")
+ .set(SQLConf.CBO_ENABLED.key, "true")
+ .set(SQLConf.PLAN_STATS_ENABLED.key, "true")
+ .set(SQLConf.JOIN_REORDER_ENABLED.key, "true")
} else {
- super.sparkConf.set(SQLConf.MAX_TO_STRING_FIELDS, Int.MaxValue)
+ super.sparkConf.set(SQLConf.MAX_TO_STRING_FIELDS.key, s"${Int.MaxValue}")
}
}
diff --git
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenTPCHBase.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenTPCHBase.scala
similarity index 98%
rename from
backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenTPCHBase.scala
rename to
backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenTPCHBase.scala
index 4ed7e4513..685f185ac 100644
---
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/gluten/test/GlutenTPCHBase.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/test/GlutenTPCHBase.scala
@@ -16,6 +16,8 @@
*/
package org.apache.spark.sql.gluten.test
+import org.apache.gluten.test.GlutenTPCBase
+
import org.apache.spark.sql.catalyst.TableIdentifier
trait GlutenTPCHBase extends GlutenTPCBase {
diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md
index 54f379148..a8862f7a5 100644
--- a/docs/developers/NewToGluten.md
+++ b/docs/developers/NewToGluten.md
@@ -309,6 +309,20 @@ After the above installation, you can optionally do some
configuration in Visual
location, you might not need to change this setting.
3. Now, you can format your CMake files by right-clicking in a file and
selecting `Format Document`.
+### Add UT
+
+1. For Native Code Modifications: If you have modified native code, it is best
to use gtest to test the native code.
+ A secondary option is to add Gluten UT to ensure coverage.
+
+2. For Gluten-Related Code Modifications: If you have modified code related to
Gluten, it is preferable to add scalatest rather than JUnit.
+ Additionally, the test classes should be placed in the org.apache.gluten
package.
+
+3. For Spark-Related Code Modifications: If you have modified code related to
Spark, it is preferable to add scalatest rather than JUnit.
+ Additionally, the test classes should be placed in the org.apache.spark
package.
+
+4. Placement of Non-Native Code UTs: Ensure that unit tests for non-native
code are placed within org.apache.gluten and org.apache.spark packages.
+ This is important because the CI system runs unit tests from these two
paths in parallel. Placing tests in other paths might cause your tests to be
ignored.
+
# Debug cpp code with coredump
```bash
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
