This is an automated email from the ASF dual-hosted git repository.
MaxGekk pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new a6dc86c3df54 [SPARK-57207][SQL][FOLLOWUP] Fix StackOverflowError when
setting timestampNanosTypes.enabled via SparkConf
a6dc86c3df54 is described below
commit a6dc86c3df54199177939785dffa2fd755a7232f
Author: Stevo Mitric <[email protected]>
AuthorDate: Wed Jun 10 21:03:54 2026 +0200
[SPARK-57207][SQL][FOLLOWUP] Fix StackOverflowError when setting
timestampNanosTypes.enabled via SparkConf
### What changes were proposed in this pull request?
This PR deletes that validator and moves the requirement into the getter:
`SQLConf.timestampNanosTypesEnabled` now returns true only when
`spark.sql.types.framework.enabled` is also true.
### Why are the changes needed?
Enabling the flag the normal way crashes Spark:
```scala
val spark = SparkSession.builder()
.master("local[1]")
.config("spark.sql.timestampNanosTypes.enabled", "true")
.getOrCreate() // fine — options are applied lazily
spark.sql("SELECT 1") // java.lang.StackOverflowError
```
The validator runs while the session conf is being built (during
`mergeSparkConf`), and `SQLConf.get` asks for that same conf which starts
building it again, forever.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Tests in this PR.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Fable 5)
Closes #56431 from
stevomitric/stevomitric/fix-nanos-conf-checkvalue-recursion.
Authored-by: Stevo Mitric <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
(cherry picked from commit 0e8a75f044e27ab2d6960a41a9841d99216207ac)
Signed-off-by: Max Gekk <[email protected]>
---
.../org/apache/spark/sql/internal/SQLConf.scala | 16 ++++++-------
.../types/ops/TimestampNanosTypeOpsSuite.scala | 28 +++++++++++-----------
.../spark/sql/SparkSessionBuilderSuite.scala | 13 ++++++++++
3 files changed, 35 insertions(+), 22 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 8a54fc55ba7a..a45ae4d5c113 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -657,17 +657,11 @@ object SQLConf {
"types. Enabling this flag does not guarantee full SQL support: casts,
Parquet read, " +
"typed literals, and other operations may still fail until their
respective features " +
"are implemented. The nanosecond timestamp types are implemented
solely through the " +
- "Types Framework, so this flag can only be set to true when " +
+ "Types Framework, so this flag only takes effect when " +
s"${TYPES_FRAMEWORK_ENABLED.key} is also true.")
.version("4.3.0")
.withBindingPolicy(ConfigBindingPolicy.SESSION)
.booleanConf
- .checkValue(
- enabled => !enabled || SQLConf.get.typesFrameworkEnabled,
- "REQUIREMENT",
- _ => Map("confRequirement" ->
- (s"'${TYPES_FRAMEWORK_ENABLED.key}' must be true to enable the
nanosecond " +
- "timestamp types.")))
.createWithDefaultFunction(() => Utils.isTesting)
val EXTENDED_EXPLAIN_PROVIDERS =
buildConf("spark.sql.extendedExplainProviders")
@@ -7648,7 +7642,13 @@ class SQLConf extends Serializable with Logging with
SqlApiConf {
def typesFrameworkEnabled: Boolean = getConf(TYPES_FRAMEWORK_ENABLED)
- def timestampNanosTypesEnabled: Boolean =
getConf(TIMESTAMP_NANOS_TYPES_ENABLED)
+ // The nanos types are implemented solely through the Types Framework, so
the flag has no
+ // effect without it. The requirement is enforced here instead of in a
checkValue of
+ // TIMESTAMP_NANOS_TYPES_ENABLED: a validator must not call SQLConf.get,
because validators
+ // run inside mergeSparkConf while the session's SQLConf is still being
constructed, and the
+ // SQLConf.get lookup re-enters that construction (infinite recursion).
+ def timestampNanosTypesEnabled: Boolean =
+ getConf(TIMESTAMP_NANOS_TYPES_ENABLED) && getConf(TYPES_FRAMEWORK_ENABLED)
def dataSourceV2JoinPushdown: Boolean = getConf(DATA_SOURCE_V2_JOIN_PUSHDOWN)
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/types/ops/TimestampNanosTypeOpsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/types/ops/TimestampNanosTypeOpsSuite.scala
index 6154dff97be5..2d69729540cd 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/types/ops/TimestampNanosTypeOpsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/types/ops/TimestampNanosTypeOpsSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.types.ops
import java.time.{Instant, LocalDateTime}
-import org.apache.spark.{SparkFunSuite, SparkIllegalArgumentException,
SparkUnsupportedOperationException}
+import org.apache.spark.{SparkFunSuite, SparkUnsupportedOperationException}
import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
import org.apache.spark.sql.catalyst.encoders.{AgnosticEncoder,
ExpressionEncoder}
import
org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.{InstantNanosEncoder,
LocalDateTimeNanosEncoder, OptionEncoder}
@@ -204,19 +204,19 @@ class TimestampNanosTypeOpsSuite extends SparkFunSuite
with SQLHelper {
}
}
- test("enabling the nanos types requires the Types Framework to be enabled") {
- withSQLConf(SQLConf.TYPES_FRAMEWORK_ENABLED.key -> "false") {
- checkError(
- exception = intercept[SparkIllegalArgumentException] {
- SQLConf.get.setConfString(SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key,
"true")
- },
- condition = "INVALID_CONF_VALUE.REQUIREMENT",
- parameters = Map(
- "confName" -> SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key,
- "confValue" -> "true",
- "confRequirement" ->
- (s"'${SQLConf.TYPES_FRAMEWORK_ENABLED.key}' must be true to enable
the nanosecond " +
- "timestamp types.")))
+ test("the nanos types flag only takes effect when the Types Framework is
enabled") {
+ // Setting the flag must always succeed (a checkValue consulting
SQLConf.get would recurse
+ // into the session conf's own construction); the framework requirement is
enforced in the
+ // effective getter instead.
+ withSQLConf(
+ SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
+ SQLConf.TYPES_FRAMEWORK_ENABLED.key -> "false") {
+ assert(!SQLConf.get.timestampNanosTypesEnabled)
+ }
+ withSQLConf(
+ SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
+ SQLConf.TYPES_FRAMEWORK_ENABLED.key -> "true") {
+ assert(SQLConf.get.timestampNanosTypesEnabled)
}
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
index bbc396c879de..a2ae074d981d 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
@@ -85,6 +85,19 @@ class SparkSessionBuilderSuite extends SparkFunSuite with
Eventually {
assert(session.conf.get("some-config") == "v2")
}
+ test("enabling timestampNanosTypes via SparkConf must not break the
session") {
+ // Regression test: the conf's checkValue used to call SQLConf.get, which
re-entered the
+ // session's sqlConf lazy val while mergeSparkConf was applying this very
conf, so the
+ // first use of the session died with StackOverflowError.
+ val session = SparkSession.builder()
+ .master("local")
+ .config(SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key, value = true)
+ .config(SQLConf.TYPES_FRAMEWORK_ENABLED.key, value = true)
+ .getOrCreate()
+ assert(session.conf.get(SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key) ==
"true")
+ assert(session.sql("SELECT 1").collect() === Array(Row(1)))
+ }
+
test("use global default session") {
val session = SparkSession.builder().master("local").getOrCreate()
assert(SparkSession.builder().getOrCreate() == session)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
