This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new fb543a7 [SPARK-38306][SQL] Fix ExplainSuite,StatisticsCollectionSuite
and StringFunctionsSuite under ANSI mode
fb543a7 is described below
commit fb543a7294dc74d211dc58cbd1c1d30a1939c344
Author: Gengliang Wang <[email protected]>
AuthorDate: Thu Feb 24 13:02:37 2022 +0800
[SPARK-38306][SQL] Fix ExplainSuite,StatisticsCollectionSuite and
StringFunctionsSuite under ANSI mode
### What changes were proposed in this pull request?
Fix ExplainSuite,StatisticsCollectionSuite and StringFunctionsSuite under
ANSI mode
### Why are the changes needed?
To set up a new GA test job with ANSI mode on
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Manually turn on ANSI mode and test
Closes #35633 from gengliangwang/fixStringFunc.
Authored-by: Gengliang Wang <[email protected]>
Signed-off-by: Gengliang Wang <[email protected]>
---
.../test/scala/org/apache/spark/sql/ExplainSuite.scala | 11 +++++------
.../org/apache/spark/sql/StatisticsCollectionSuite.scala | 4 ++--
.../org/apache/spark/sql/StringFunctionsSuite.scala | 16 ++++++++++------
3 files changed, 17 insertions(+), 14 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
index 99bdfc8..67240c5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
@@ -217,8 +217,8 @@ class ExplainSuite extends ExplainSuiteHelper with
DisableAdaptiveExecutionSuite
// AND conjunction
// OR disjunction
//
---------------------------------------------------------------------------------------
- checkKeywordsExistsInExplain(sql("select 'a' || 1 + 2"),
- "Project [null AS (concat(a, 1) + 2)#x]")
+ checkKeywordsExistsInExplain(sql("select '1' || 1 + 2"),
+ "Project [13", " AS (concat(1, 1) + 2)#x")
checkKeywordsExistsInExplain(sql("select 1 - 2 || 'b'"),
"Project [-1b AS concat((1 - 2), b)#x]")
checkKeywordsExistsInExplain(sql("select 2 * 4 + 3 || 'b'"),
@@ -232,12 +232,11 @@ class ExplainSuite extends ExplainSuiteHelper with
DisableAdaptiveExecutionSuite
}
test("explain for these functions; use range to avoid constant folding") {
- val df = sql("select ifnull(id, 'x'), nullif(id, 'x'), nvl(id, 'x'),
nvl2(id, 'x', 'y') " +
+ val df = sql("select ifnull(id, 1), nullif(id, 1), nvl(id, 1), nvl2(id, 1,
2) " +
"from range(2)")
checkKeywordsExistsInExplain(df,
- "Project [cast(id#xL as string) AS ifnull(id, x)#x, " +
- "id#xL AS nullif(id, x)#xL, cast(id#xL as string) AS nvl(id, x)#x, " +
- "x AS nvl2(id, x, y)#x]")
+ "Project [id#xL AS ifnull(id, 1)#xL, if ((id#xL = 1)) null " +
+ "else id#xL AS nullif(id, 1)#xL, id#xL AS nvl(id, 1)#xL, 1 AS nvl2(id,
1, 2)#x]")
}
test("SPARK-26659: explain of DataWritingCommandExec should not contain
duplicate cmd.nodeName") {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
index 0987825..57fc49d 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
@@ -409,7 +409,7 @@ class StatisticsCollectionSuite extends
StatisticsCollectionTestBase with Shared
val df = spark.range(1000L).select('id,
'id * 2 as "FLD1",
'id * 12 as "FLD2",
- lit("aaa") + 'id as "fld3")
+ lit(null).cast(DoubleType) + 'id as "fld3")
df.write
.mode(SaveMode.Overwrite)
.bucketBy(10, "id", "FLD1", "FLD2")
@@ -425,7 +425,7 @@ class StatisticsCollectionSuite extends
StatisticsCollectionTestBase with Shared
|WHERE t1.fld3 IN (-123.23,321.23)
""".stripMargin)
df2.createTempView("TBL2")
- sql("SELECT * FROM tbl2 WHERE fld3 IN ('qqq', 'qwe')
").queryExecution.executedPlan
+ sql("SELECT * FROM tbl2 WHERE fld3 IN (0,1)
").queryExecution.executedPlan
}
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
index 30a6600..2f118f2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
@@ -112,9 +112,11 @@ class StringFunctionsSuite extends QueryTest with
SharedSparkSession {
val df = Seq[(String, String, String, Int)](("hello", "world", null, 15))
.toDF("a", "b", "c", "d")
- checkAnswer(
- df.selectExpr("elt(0, a, b, c)", "elt(1, a, b, c)", "elt(4, a, b, c)"),
- Row(null, "hello", null))
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+ checkAnswer(
+ df.selectExpr("elt(0, a, b, c)", "elt(1, a, b, c)", "elt(4, a, b, c)"),
+ Row(null, "hello", null))
+ }
// check implicit type cast
checkAnswer(
@@ -383,9 +385,11 @@ class StringFunctionsSuite extends QueryTest with
SharedSparkSession {
Row("host", "/file;param", "query;p2", null, "http",
"/file;param?query;p2",
"user:pass@host", "user:pass", null))
- testUrl(
- "inva lid://user:pass@host/file;param?query;p2",
- Row(null, null, null, null, null, null, null, null, null))
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+ testUrl(
+ "inva lid://user:pass@host/file;param?query;p2",
+ Row(null, null, null, null, null, null, null, null, null))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
