This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9d27db940673 [SPARK-53474][SQL] Add conf DATA_SOURCE_V2_EXPR_FOLDING
9d27db940673 is described below
commit 9d27db940673196f64b1568d73c459f83eb6f788
Author: yhuang-db <itisyuch...@gmail.com>
AuthorDate: Fri Sep 5 13:53:04 2025 +0800
[SPARK-53474][SQL] Add conf DATA_SOURCE_V2_EXPR_FOLDING
### What changes were proposed in this pull request?
This PR introduces `spark.sql.optimizer.datasourceV2ExprFolding`, a SQL
conf that when it is set to true, do safe constant folding for the expressions
before translation and pushdown. The conf is set to `true` by default.
https://github.com/apache/spark/pull/51569 is the original PR adding dsv2
expression folding.
### Why are the changes needed?
This conf governs whether an expression will be folded to constant. Also
for test compatibility.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing tests.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #52222 from yhuang-db/add_conf.
Authored-by: yhuang-db <itisyuch...@gmail.com>
Signed-off-by: Wenchen Fan <wenc...@databricks.com>
---
.../spark/sql/catalyst/util/V2ExpressionBuilder.scala | 3 ++-
.../main/scala/org/apache/spark/sql/internal/SQLConf.scala | 8 ++++++++
.../datasources/v2/DataSourceV2StrategySuite.scala | 14 ++++++++++++++
3 files changed, 24 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala
index e42802de40f2..4e391208d984 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala
@@ -93,7 +93,8 @@ class V2ExpressionBuilder(e: Expression, isPredicate: Boolean
= false) extends L
private def generateExpression(
expr: Expression, isPredicate: Boolean = false): Option[V2Expression] =
expr match {
case literal: Literal => Some(translateLiteral(literal))
- case _ if expr.contextIndependentFoldable =>
+ case _ if expr.contextIndependentFoldable
+ && SQLConf.get.getConf(SQLConf.DATA_SOURCE_V2_EXPR_FOLDING) =>
// If the expression is context independent foldable, we can convert it
to a literal.
// This is useful for increasing the coverage of V2 expressions.
val constantExpr = ConstantFolding.constantFolding(expr)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 1584f05ac476..12fd80c5a626 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1834,6 +1834,14 @@ object SQLConf {
.booleanConf
.createWithDefault(false)
+ val DATA_SOURCE_V2_EXPR_FOLDING =
+ buildConf("spark.sql.optimizer.datasourceV2ExprFolding")
+ .internal()
+ .doc("When this config is set to true, do safe constant folding for the
" +
+ "expressions before translation and pushdown.")
+ .booleanConf
+ .createWithDefault(true)
+
// This is used to set the default data source
val DEFAULT_DATA_SOURCE_NAME = buildConf("spark.sql.sources.default")
.doc("The default data source to use in input/output.")
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala
index 4c6b0a54c818..ab6668eeb45a 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2StrategySuite.scala
@@ -831,6 +831,20 @@ class DataSourceV2StrategySuite extends PlanTest with
SharedSparkSession {
}
}
+ test("SPARK-53474: Check failure when datasourceV2ExprFolding = false") {
+ // when spark.sql.optimizer.datasourceV2ExprFolding = true
+ // expression will first convert to V2 expressions, then fold to constant
+ val expr = Abs(Literal(-5), failOnError = true)
+ checkV2Conversion(expr, LiteralValue(5, IntegerType))
+
+ withSQLConf(SQLConf.DATA_SOURCE_V2_EXPR_FOLDING.key -> "false") {
+ // when spark.sql.optimizer.datasourceV2ExprFolding = false
+ // expression will be converted to V2 expressions, but not folded
+ checkV2Conversion(expr,
+ new GeneralScalarExpression("ABS", Array(LiteralValue(-5,
IntegerType))))
+ }
+ }
+
/**
* Translate the given Catalyst [[Expression]] into data source V2
[[Predicate]]
* then verify against the given [[Predicate]].
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
