This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 9dacbc0 [SPARK-38432][SQL][FOLLOWUP] Supplement test case for
overflow and add comments
9dacbc0 is described below
commit 9dacbc0ffa6bfe062abbc479d109248640976897
Author: Jiaan Geng <belie...@163.com>
AuthorDate: Wed Mar 23 09:47:35 2022 +0800
[SPARK-38432][SQL][FOLLOWUP] Supplement test case for overflow and add
comments
### What changes were proposed in this pull request?
This PR follows up https://github.com/apache/spark/pull/35768 and improves
the code.
1. Supplement test case for overflow
2. Not throw IllegalArgumentException
3. Improve V2ExpressionSQLBuilder
4. Add comments in V2ExpressionBuilder
### Why are the changes needed?
Supplement test case for overflow and add comments.
### Does this PR introduce _any_ user-facing change?
'No'.
V2 aggregate pushdown not released yet.
### How was this patch tested?
New tests.
Closes #35933 from beliefer/SPARK-38432_followup.
Authored-by: Jiaan Geng <belie...@163.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
(cherry picked from commit 4e606383a663919b7120789ae741a0f6698e3ff0)
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../sql/connector/util/V2ExpressionSQLBuilder.java | 6 ++--
.../sql/catalyst/util/V2ExpressionBuilder.scala | 2 ++
.../org/apache/spark/sql/jdbc/JdbcDialects.scala | 5 ++--
.../org/apache/spark/sql/jdbc/JDBCV2Suite.scala | 34 ++++++++++++++++++----
4 files changed, 36 insertions(+), 11 deletions(-)
diff --git
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java
index 91dae74..1df01d2 100644
---
a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java
+++
b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java
@@ -75,7 +75,7 @@ public class V2ExpressionSQLBuilder {
name, inputToSQL(e.children()[0]), inputToSQL(e.children()[1]));
case "-":
if (e.children().length == 1) {
- return visitUnaryArithmetic(name, build(e.children()[0]));
+ return visitUnaryArithmetic(name, inputToSQL(e.children()[0]));
} else {
return visitBinaryArithmetic(
name, inputToSQL(e.children()[0]), inputToSQL(e.children()[1]));
@@ -87,7 +87,7 @@ public class V2ExpressionSQLBuilder {
case "NOT":
return visitNot(build(e.children()[0]));
case "~":
- return visitUnaryArithmetic(name, build(e.children()[0]));
+ return visitUnaryArithmetic(name, inputToSQL(e.children()[0]));
case "CASE_WHEN": {
List<String> children =
Arrays.stream(e.children()).map(c ->
build(c)).collect(Collectors.toList());
@@ -179,7 +179,7 @@ public class V2ExpressionSQLBuilder {
return "NOT (" + v + ")";
}
- protected String visitUnaryArithmetic(String name, String v) { return name
+" (" + v + ")"; }
+ protected String visitUnaryArithmetic(String name, String v) { return name +
v; }
protected String visitCaseWhen(String[] children) {
StringBuilder sb = new StringBuilder("CASE");
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala
index 5c8e6a6..fbd6884 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala
@@ -94,6 +94,7 @@ class V2ExpressionBuilder(
None
}
case and: And =>
+ // AND expects predicate
val l = generateExpression(and.left, true)
val r = generateExpression(and.right, true)
if (l.isDefined && r.isDefined) {
@@ -103,6 +104,7 @@ class V2ExpressionBuilder(
None
}
case or: Or =>
+ // OR expects predicate
val l = generateExpression(or.left, true)
val r = generateExpression(or.right, true)
if (l.isDefined && r.isDefined) {
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
index 4b28de2..674ef00 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
@@ -229,9 +229,8 @@ abstract class JdbcDialect extends Serializable with
Logging{
override def visitNamedReference(namedRef: NamedReference): String = {
if (namedRef.fieldNames().length > 1) {
- throw new IllegalArgumentException(
- QueryCompilationErrors.commandNotSupportNestedColumnError(
- "Filter push down", namedRef.toString).getMessage);
+ throw QueryCompilationErrors.commandNotSupportNestedColumnError(
+ "Filter push down", namedRef.toString)
}
quoteIdentifier(namedRef.fieldNames.head)
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
index d50a055..d6f098f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
@@ -402,14 +402,38 @@ class JDBCV2Suite extends QueryTest with
SharedSparkSession with ExplainSuiteHel
checkAnswer(df, Seq(Row("fred", 1), Row("mary", 2)))
- val df2 = sql("""
+ val df2 = spark.table("h2.test.people").filter($"id" + Int.MaxValue >
1)
+
+ checkFiltersRemoved(df2, ansiMode)
+
+ df2.queryExecution.optimizedPlan.collect {
+ case _: DataSourceV2ScanRelation =>
+ val expected_plan_fragment = if (ansiMode) {
+ "PushedFilters: [ID IS NOT NULL, (ID + 2147483647) > 1], "
+ } else {
+ "PushedFilters: [ID IS NOT NULL], "
+ }
+ checkKeywordsExistsInExplain(df2, expected_plan_fragment)
+ }
+
+ if (ansiMode) {
+ val e = intercept[SparkException] {
+ checkAnswer(df2, Seq.empty)
+ }
+ assert(e.getMessage.contains(
+ "org.h2.jdbc.JdbcSQLDataException: Numeric value out of range:
\"2147483648\""))
+ } else {
+ checkAnswer(df2, Seq.empty)
+ }
+
+ val df3 = sql("""
|SELECT * FROM h2.test.employee
|WHERE (CASE WHEN SALARY > 10000 THEN BONUS ELSE BONUS
+ 200 END) > 1200
|""".stripMargin)
- checkFiltersRemoved(df2, ansiMode)
+ checkFiltersRemoved(df3, ansiMode)
- df2.queryExecution.optimizedPlan.collect {
+ df3.queryExecution.optimizedPlan.collect {
case _: DataSourceV2ScanRelation =>
val expected_plan_fragment = if (ansiMode) {
"PushedFilters: [(CASE WHEN SALARY > 10000.00 THEN BONUS" +
@@ -417,10 +441,10 @@ class JDBCV2Suite extends QueryTest with
SharedSparkSession with ExplainSuiteHel
} else {
"PushedFilters: []"
}
- checkKeywordsExistsInExplain(df2, expected_plan_fragment)
+ checkKeywordsExistsInExplain(df3, expected_plan_fragment)
}
- checkAnswer(df2,
+ checkAnswer(df3,
Seq(Row(1, "cathy", 9000, 1200, false), Row(2, "david", 10000, 1300,
true)))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
