This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new e9887fb306 [GLUTEN-8633][VL] Rewrite Spark tests for Gluten
ColumnarRange (#8634)
e9887fb306 is described below
commit e9887fb3066bb9d06ef6ddcf4cb86310509ad8f0
Author: Arnav Balyan <[email protected]>
AuthorDate: Mon Mar 10 22:25:52 2025 +0530
[GLUTEN-8633][VL] Rewrite Spark tests for Gluten ColumnarRange (#8634)
---
.../utils/clickhouse/ClickHouseTestSettings.scala | 2 ++
.../apache/spark/sql/GlutenDataFrameSuite.scala | 24 ++++++++++++++++++++++
.../utils/clickhouse/ClickHouseTestSettings.scala | 2 ++
.../apache/spark/sql/GlutenDataFrameSuite.scala | 24 ++++++++++++++++++++++
.../utils/clickhouse/ClickHouseTestSettings.scala | 2 ++
.../apache/spark/sql/GlutenDataFrameSuite.scala | 24 ++++++++++++++++++++++
.../utils/clickhouse/ClickHouseTestSettings.scala | 2 ++
.../apache/spark/sql/GlutenDataFrameSuite.scala | 24 ++++++++++++++++++++++
8 files changed, 104 insertions(+)
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 3d79265561..ca7749735b 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -242,6 +242,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("SPARK-24165: CaseWhen/If - nullability of nested types")
.exclude("SPARK-27671: Fix analysis exception when casting null in nested
field in struct")
.exclude("summary")
+ .excludeGlutenTest(
+ "SPARK-27439: Explain result should match collected result after view
change")
.excludeGlutenTest("distributeBy and localSort")
.excludeGlutenTest("describe")
.excludeGlutenTest("Allow leading/trailing whitespace in string before
casting")
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
index 50deb211d1..fd2ac1408d 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestData.TestData2
import org.apache.spark.sql.types.StringType
+import java.io.ByteArrayOutputStream
import java.nio.charset.StandardCharsets
import scala.util.Random
@@ -359,6 +360,29 @@ class GlutenDataFrameSuite extends DataFrameSuite with
GlutenSQLTestsTrait {
checkResult(df, expectedBinaryResult)
}
+ testGluten("SPARK-27439: Explain result should match collected result after
view change") {
+ withTempView("test", "test2", "tmp") {
+ spark.range(10).createOrReplaceTempView("test")
+ spark.range(5).createOrReplaceTempView("test2")
+ spark.sql("select * from test").createOrReplaceTempView("tmp")
+ val df = spark.sql("select * from tmp")
+ spark.sql("select * from test2").createOrReplaceTempView("tmp")
+
+ val captured = new ByteArrayOutputStream()
+ Console.withOut(captured) {
+ df.explain(extended = true)
+ }
+ checkAnswer(df, spark.range(10).toDF)
+ val output = captured.toString
+ assert(output.contains("""== Parsed Logical Plan ==
+ |'Project [*]
+ |+- 'UnresolvedRelation [tmp]""".stripMargin))
+ assert(output.contains("""== Physical Plan ==
+ |*(1) ColumnarToRow
+ |+- ColumnarRange 0, 10, 1, 2,
10""".stripMargin))
+ }
+ }
+
private def withExpr(newExpr: Expression): Column = new Column(newExpr)
def equalizer(expr: Expression, other: Any): Column = withExpr {
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 8819991e81..86ad1b983a 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -264,6 +264,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("SPARK-24165: CaseWhen/If - nullability of nested types")
.exclude("SPARK-27671: Fix analysis exception when casting null in nested
field in struct")
.exclude("summary")
+ .excludeGlutenTest(
+ "SPARK-27439: Explain result should match collected result after view
change")
.excludeGlutenTest("distributeBy and localSort")
.excludeGlutenTest("describe")
.excludeGlutenTest("Allow leading/trailing whitespace in string before
casting")
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
index 4008f862e1..871cc98d5f 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestData.TestData2
import org.apache.spark.sql.types.StringType
+import java.io.ByteArrayOutputStream
import java.nio.charset.StandardCharsets
import scala.util.Random
@@ -360,6 +361,29 @@ class GlutenDataFrameSuite extends DataFrameSuite with
GlutenSQLTestsTrait {
checkResult(df, expectedBinaryResult)
}
+ testGluten("SPARK-27439: Explain result should match collected result after
view change") {
+ withTempView("test", "test2", "tmp") {
+ spark.range(10).createOrReplaceTempView("test")
+ spark.range(5).createOrReplaceTempView("test2")
+ spark.sql("select * from test").createOrReplaceTempView("tmp")
+ val df = spark.sql("select * from tmp")
+ spark.sql("select * from test2").createOrReplaceTempView("tmp")
+
+ val captured = new ByteArrayOutputStream()
+ Console.withOut(captured) {
+ df.explain(extended = true)
+ }
+ checkAnswer(df, spark.range(10).toDF)
+ val output = captured.toString
+ assert(output.contains("""== Parsed Logical Plan ==
+ |'Project [*]
+ |+- 'UnresolvedRelation [tmp]""".stripMargin))
+ assert(output.contains("""== Physical Plan ==
+ |*(1) ColumnarToRow
+ |+- ColumnarRange 0, 10, 1, 2,
10""".stripMargin))
+ }
+ }
+
private def withExpr(newExpr: Expression): Column = new Column(newExpr)
def equalizer(expr: Expression, other: Any): Column = withExpr {
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 434addc6bc..afc3b7f6b2 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -262,6 +262,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
.exclude("SPARK-24165: CaseWhen/If - nullability of nested types")
.exclude("SPARK-27671: Fix analysis exception when casting null in nested
field in struct")
.exclude("summary")
+ .excludeGlutenTest(
+ "SPARK-27439: Explain result should match collected result after view
change")
.excludeGlutenTest("distributeBy and localSort")
.excludeGlutenTest("describe")
.excludeGlutenTest("Allow leading/trailing whitespace in string before
casting")
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
index 4008f862e1..871cc98d5f 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestData.TestData2
import org.apache.spark.sql.types.StringType
+import java.io.ByteArrayOutputStream
import java.nio.charset.StandardCharsets
import scala.util.Random
@@ -360,6 +361,29 @@ class GlutenDataFrameSuite extends DataFrameSuite with
GlutenSQLTestsTrait {
checkResult(df, expectedBinaryResult)
}
+ testGluten("SPARK-27439: Explain result should match collected result after
view change") {
+ withTempView("test", "test2", "tmp") {
+ spark.range(10).createOrReplaceTempView("test")
+ spark.range(5).createOrReplaceTempView("test2")
+ spark.sql("select * from test").createOrReplaceTempView("tmp")
+ val df = spark.sql("select * from tmp")
+ spark.sql("select * from test2").createOrReplaceTempView("tmp")
+
+ val captured = new ByteArrayOutputStream()
+ Console.withOut(captured) {
+ df.explain(extended = true)
+ }
+ checkAnswer(df, spark.range(10).toDF)
+ val output = captured.toString
+ assert(output.contains("""== Parsed Logical Plan ==
+ |'Project [*]
+ |+- 'UnresolvedRelation [tmp]""".stripMargin))
+ assert(output.contains("""== Physical Plan ==
+ |*(1) ColumnarToRow
+ |+- ColumnarRange 0, 10, 1, 2,
10""".stripMargin))
+ }
+ }
+
private def withExpr(newExpr: Expression): Column = new Column(newExpr)
def equalizer(expr: Expression, other: Any): Column = withExpr {
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
index 75e97b48a9..ac4b11ee1b 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala
@@ -546,6 +546,8 @@ class ClickHouseTestSettings extends BackendTestSettings {
.excludeCH("SPARK-28067: Aggregate sum should not return wrong results for
decimal overflow")
.excludeCH("SPARK-35955: Aggregate avg should not return wrong results for
decimal overflow")
.excludeCH("summary")
+ .excludeGlutenTest(
+ "SPARK-27439: Explain result should match collected result after view
change")
.excludeCH(
"SPARK-8608: call `show` on local DataFrame with random columns should
return same value")
.excludeCH(
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
index bfd12cf532..6b35c89287 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDataFrameSuite.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestData.TestData2
import org.apache.spark.sql.types.StringType
+import java.io.ByteArrayOutputStream
import java.nio.charset.StandardCharsets
import scala.util.Random
@@ -371,6 +372,29 @@ class GlutenDataFrameSuite extends DataFrameSuite with
GlutenSQLTestsTrait {
}
}
+ testGluten("SPARK-27439: Explain result should match collected result after
view change") {
+ withTempView("test", "test2", "tmp") {
+ spark.range(10).createOrReplaceTempView("test")
+ spark.range(5).createOrReplaceTempView("test2")
+ spark.sql("select * from test").createOrReplaceTempView("tmp")
+ val df = spark.sql("select * from tmp")
+ spark.sql("select * from test2").createOrReplaceTempView("tmp")
+
+ val captured = new ByteArrayOutputStream()
+ Console.withOut(captured) {
+ df.explain(extended = true)
+ }
+ checkAnswer(df, spark.range(10).toDF)
+ val output = captured.toString
+ assert(output.contains("""== Parsed Logical Plan ==
+ |'Project [*]
+ |+- 'UnresolvedRelation [tmp]""".stripMargin))
+ assert(output.contains("""== Physical Plan ==
+ |*(1) ColumnarToRow
+ |+- ColumnarRange 0, 10, 1, 2,
10""".stripMargin))
+ }
+ }
+
private def withExpr(newExpr: Expression): Column = new Column(newExpr)
def equalizer(expr: Expression, other: Any): Column = withExpr {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
