This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 0d78ac2fc [GLUTEN-5381] Refine testWithSpecifiedSparkVersion to
compare major version (#5382)
0d78ac2fc is described below
commit 0d78ac2fcc8e81fdd2bf069bd973c65a3382e4c6
Author: Yan Ma <[email protected]>
AuthorDate: Mon Apr 15 09:55:10 2024 +0800
[GLUTEN-5381] Refine testWithSpecifiedSparkVersion to compare major version
(#5382)
---
.../execution/ScalarFunctionsValidateSuite.scala | 4 +--
.../org/apache/spark/sql/GlutenQueryTest.scala | 34 +++++++++++++++++-----
.../apache/gluten/execution/VeloxDeltaSuite.scala | 8 ++---
3 files changed, 32 insertions(+), 14 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index 6fceb7364..daf8fa269 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -550,7 +550,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateTest {
}
}
- testWithSpecifiedSparkVersion("Test url_decode function", Some("3.4.2")) {
+ testWithSpecifiedSparkVersion("Test url_decode function", Some("3.4")) {
withTempPath {
path =>
Seq("https%3A%2F%2Fspark.apache.org")
@@ -565,7 +565,7 @@ class ScalarFunctionsValidateSuite extends
FunctionsValidateTest {
}
}
- testWithSpecifiedSparkVersion("Test url_encode function", Some("3.4.2")) {
+ testWithSpecifiedSparkVersion("Test url_encode function", Some("3.4")) {
withTempPath {
path =>
Seq("https://spark.apache.org";)
diff --git
a/gluten-core/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala
b/gluten-core/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala
index d6a2a0c63..93ae5cd36 100644
--- a/gluten-core/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala
+++ b/gluten-core/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala
@@ -44,14 +44,35 @@ abstract class GlutenQueryTest extends PlanTest {
protected def spark: SparkSession
+ def shouldRun(
+ minSparkVersion: Option[String] = None,
+ maxSparkVersion: Option[String] = None): Boolean = {
+ val version = SPARK_VERSION_SHORT.split("\\.")
+ var shouldRun = true
+ if (!minSparkVersion.isEmpty) {
+ val minVersion = minSparkVersion.get.split("\\.");
+ shouldRun =
+ minVersion(0) < version(0) || (minVersion(0) == version(0) &&
minVersion(1) <= version(1))
+ if (!maxSparkVersion.isEmpty) {
+ val maxVersion = maxSparkVersion.get.split("\\.")
+ shouldRun = shouldRun && (maxVersion(0) > version(0) || maxVersion(0)
== version(
+ 0) && maxVersion(1) >= version(1))
+ }
+ } else {
+ if (!maxSparkVersion.isEmpty) {
+ val maxVersion = maxSparkVersion.get.split("\\.")
+ shouldRun =
+ maxVersion(0) > version(0) || maxVersion(0) == version(0) &&
maxVersion(1) >= version(1)
+ }
+ }
+ shouldRun
+ }
+
def ignore(
testName: String,
minSparkVersion: Option[String] = None,
maxSparkVersion: Option[String] = None)(testFun: => Any): Unit = {
- if (
- minSparkVersion.forall(_ <= SPARK_VERSION_SHORT)
- && maxSparkVersion.forall(_ >= SPARK_VERSION_SHORT)
- ) {
+ if (shouldRun(minSparkVersion, maxSparkVersion)) {
ignore(testName) {
testFun
}
@@ -62,10 +83,7 @@ abstract class GlutenQueryTest extends PlanTest {
testName: String,
minSparkVersion: Option[String] = None,
maxSparkVersion: Option[String] = None)(testFun: => Any): Unit = {
- if (
- minSparkVersion.forall(_ <= SPARK_VERSION_SHORT)
- && maxSparkVersion.forall(_ >= SPARK_VERSION_SHORT)
- ) {
+ if (shouldRun(minSparkVersion, maxSparkVersion)) {
test(testName) {
testFun
}
diff --git
a/gluten-delta/src/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala
b/gluten-delta/src/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala
index 9347f7fa5..8cf4baae8 100644
---
a/gluten-delta/src/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala
+++
b/gluten-delta/src/test/scala/org/apache/gluten/execution/VeloxDeltaSuite.scala
@@ -43,7 +43,7 @@ class VeloxDeltaSuite extends WholeStageTransformerSuite {
// IdMapping is supported in Delta 2.2 (related to Spark3.3.1)
// Disable for Spark3.5.
- testWithSpecifiedSparkVersion("column mapping mode = id", Some("3.3.1"),
Some("3.4")) {
+ testWithSpecifiedSparkVersion("column mapping mode = id", Some("3.3"),
Some("3.4")) {
withTable("delta_cm1") {
spark.sql(s"""
|create table delta_cm1 (id int, name string) using delta
@@ -64,7 +64,7 @@ class VeloxDeltaSuite extends WholeStageTransformerSuite {
// NameMapping is supported in Delta 2.0 (related to Spark3.2.0)
// Disable for Spark3.5.
- testWithSpecifiedSparkVersion("column mapping mode = name", Some("3.2.0"),
Some("3.4")) {
+ testWithSpecifiedSparkVersion("column mapping mode = name", Some("3.2"),
Some("3.4")) {
withTable("delta_cm2") {
spark.sql(s"""
|create table delta_cm2 (id int, name string) using delta
@@ -84,7 +84,7 @@ class VeloxDeltaSuite extends WholeStageTransformerSuite {
}
// Disable for Spark3.5.
- testWithSpecifiedSparkVersion("delta: time travel", Some("3.3.1"),
Some("3.4")) {
+ testWithSpecifiedSparkVersion("delta: time travel", Some("3.3"),
Some("3.4")) {
withTable("delta_tm") {
spark.sql(s"""
|create table delta_tm (id int, name string) using delta
@@ -205,7 +205,7 @@ class VeloxDeltaSuite extends WholeStageTransformerSuite {
}
// Disable for Spark3.5
- testWithSpecifiedSparkVersion("deletion vector", Some("3.4.2"),
Some("3.4.2")) {
+ testWithSpecifiedSparkVersion("deletion vector", Some("3.4"), Some("3.4")) {
withTempPath {
p =>
import testImplicits._
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
