This is an automated email from the ASF dual-hosted git repository.
MaxGekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 0c689aed7d6c [SPARK-57555][SQL][FOLLOWUP] Add escape-hatch conf to
keep legacy JDBC TIME-to-timestamp mapping
0c689aed7d6c is described below
commit 0c689aed7d6cd82d20a68ce4faea227e770f71f4
Author: Wenchen Fan <[email protected]>
AuthorDate: Wed Jul 1 09:11:33 2026 +0200
[SPARK-57555][SQL][FOLLOWUP] Add escape-hatch conf to keep legacy JDBC
TIME-to-timestamp mapping
### What changes were proposed in this pull request?
[SPARK-57555](https://issues.apache.org/jira/browse/SPARK-57555) made the
built-in JDBC data source read SQL `TIME` columns as `TimeType` (instead of the
legacy `TimestampType`) when `spark.sql.timeType.enabled` is true. That is a
schema-inference change for existing JDBC reads: a column that previously came
back as `TimestampType` now comes back as `TimeType`, with no DDL change by the
user.
This PR adds an internal escape-hatch conf,
`spark.sql.legacy.jdbc.timeMapping.enabled` (default `false`), following the
existing `spark.sql.legacy.<db>.*Mapping.enabled` family. When set to true,
JDBC `TIME` columns keep the legacy `TimestampType` mapping even when
`spark.sql.timeType.enabled` is true. It has no effect when
`spark.sql.timeType.enabled` is false.
### Why are the changes needed?
`spark.sql.timeType.enabled` is a single global switch that governs `TIME`
support across all of Spark. Enabling it flips JDBC `TIME` reads from
`TimestampType` to `TimeType` in one step, which can silently break workloads
that depend on the old mapping (downstream casts, comparisons, stored schemas,
serialization). The escape hatch lets such workloads opt back into the legacy
JDBC behavior without having to keep the entire `TIME` type disabled.
### Does this PR introduce any user-facing change?
No behavioral change by default. The new conf defaults to `false`,
preserving SPARK-57555 behavior. When explicitly set to `true` (and
`spark.sql.timeType.enabled` is `true`), JDBC `TIME` columns read as
`TimestampType` as they did before SPARK-57555.
### How was this patch tested?
Added a unit test in `JDBCSuite` asserting that, with
`spark.sql.timeType.enabled=true`, the column reads as `TimestampType` when the
escape hatch is on and as `TimeType` when it is off.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Opus 4.8)
Closes #56884 from cloud-fan/dw-jdbc-time-type-legacy-conf.
Authored-by: Wenchen Fan <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
---
.../org/apache/spark/sql/internal/SQLConf.scala | 16 +++++++++++++
.../sql/execution/datasources/jdbc/JdbcUtils.scala | 2 +-
.../org/apache/spark/sql/jdbc/JDBCSuite.scala | 26 ++++++++++++++++++++++
3 files changed, 43 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 6776f88ed1ef..91635d346605 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -6289,6 +6289,19 @@ object SQLConf {
.booleanConf
.createWithDefault(false)
+ val LEGACY_JDBC_TIME_MAPPING_ENABLED =
+ buildConf("spark.sql.legacy.jdbc.timeMapping.enabled")
+ .internal()
+ .doc("When true, JDBC TIME columns are read as TimestampType (the legacy
behavior), even " +
+ "when spark.sql.timeType.enabled is true. This is an escape hatch for
workloads that " +
+ "rely on the old TIME-to-timestamp mapping. When false, JDBC TIME
columns are read as " +
+ "TimeType if spark.sql.timeType.enabled is true. Has no effect when " +
+ "spark.sql.timeType.enabled is false.")
+ .version("4.3.0")
+ .withBindingPolicy(ConfigBindingPolicy.SESSION)
+ .booleanConf
+ .createWithDefault(false)
+
val PYTHON_FILTER_PUSHDOWN_ENABLED =
buildConf("spark.sql.python.filterPushdown.enabled")
.internal()
.doc("When true, enable filter pushdown to Python datasource, at the cost
of running " +
@@ -8156,6 +8169,9 @@ class SQLConf extends Serializable with Logging with
SqlApiConf {
def legacyPostgresDatetimeMappingEnabled: Boolean =
getConf(LEGACY_POSTGRES_DATETIME_MAPPING_ENABLED)
+ def legacyJdbcTimeMappingEnabled: Boolean =
+ getConf(LEGACY_JDBC_TIME_MAPPING_ENABLED)
+
override def legacyTimeParserPolicy: LegacyBehaviorPolicy.Value =
getConf(SQLConf.LEGACY_TIME_PARSER_POLICY)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
index e4f2dafa9aad..cebc45002d44 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
@@ -231,7 +231,7 @@ object JdbcUtils extends Logging with SQLConfHelper {
case java.sql.Types.SQLXML => StringType
case java.sql.Types.STRUCT => StringType
case java.sql.Types.TIME =>
- if (conf.isTimeTypeEnabled) {
+ if (conf.isTimeTypeEnabled && !conf.legacyJdbcTimeMappingEnabled) {
// Use reported scale (fractional digits) as precision; TIME(0) is
valid
val timePrecision = if (scale >= 0 && scale <= TimeType.MAX_PRECISION)
scale
else TimeType.DEFAULT_PRECISION
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index 4e75e1807cf4..f353f6f2a699 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -768,6 +768,32 @@ class JDBCSuite extends SharedSparkSession {
assert(rows(0).getAs[java.time.LocalTime](0) ===
java.time.LocalTime.of(12, 34, 56))
}
+ test("SPARK-57555: legacy.jdbc.timeMapping escape hatch keeps TIME as
TimestampType") {
+ // The escape hatch forces the legacy TIME-to-timestamp mapping even when
the TIME type is
+ // enabled, so workloads relying on the old behavior are not silently
broken.
+ withSQLConf(
+ SQLConf.TIME_TYPE_ENABLED.key -> "true",
+ SQLConf.LEGACY_JDBC_TIME_MAPPING_ENABLED.key -> "true") {
+ val df = spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new
Properties())
+ assert(df.schema("A").dataType === TimestampType)
+ }
+ // Sanity check: without the escape hatch the column is read as TimeType.
+ withSQLConf(
+ SQLConf.TIME_TYPE_ENABLED.key -> "true",
+ SQLConf.LEGACY_JDBC_TIME_MAPPING_ENABLED.key -> "false") {
+ val df = spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new
Properties())
+ assert(df.schema("A").dataType.isInstanceOf[TimeType])
+ }
+ // The escape hatch has no effect when the TIME type is disabled: the
column is read as
+ // TimestampType regardless of the escape hatch.
+ withSQLConf(
+ SQLConf.TIME_TYPE_ENABLED.key -> "false",
+ SQLConf.LEGACY_JDBC_TIME_MAPPING_ENABLED.key -> "true") {
+ val df = spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new
Properties())
+ assert(df.schema("A").dataType === TimestampType)
+ }
+ }
+
test("SPARK-57555: JDBC TIME write round-trip") {
val url = urlWithUserAndPass
val tableName = "TEST.TIME_ROUNDTRIP"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
