This is an automated email from the ASF dual-hosted git repository.
LuciferYang pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 9b288254ba9d [SPARK-58296][SQL] Fix to_time returning STRING type when
the format is a foldable NULL
9b288254ba9d is described below
commit 9b288254ba9d276c173181e1bd7ba369d68bf13d
Author: YangJie <[email protected]>
AuthorDate: Mon Jul 27 11:36:46 2026 +0800
[SPARK-58296][SQL] Fix to_time returning STRING type when the format is a
foldable NULL
### What changes were proposed in this pull request?
`ToTime` (the `to_time` function) is a `RuntimeReplaceable`, so its
`dataType` comes from `replacement.dataType`. When the format argument is
foldable and evaluates to NULL, the replacement fell back to `Literal(null,
expr.dataType)`, where `expr` is the format argument, so the whole expression
reported the format's type (STRING) instead of TIME. This changes the fallback
to `Literal(null, TimeType())`, so `to_time` reports `TimeType()` like all of
its other branches.
### Why are the changes needed?
`to_time(str, fmt)` with a foldable NULL `fmt` reports the wrong type.
`SELECT typeof(to_time('00:12:00', null))` returns `string` instead of
`time(6)`, and the analyzed plan carries STRING for that column, so schema
inference and UNION type coercion (against a real TIME column) use the wrong
type. The value is NULL either way, but the type is user-visible and wrong. The
inconsistency is also internal: a non-foldable NULL format takes the
`invokeParser()` branch and correctly yields ` [...]
### Does this PR introduce _any_ user-facing change?
Yes. `to_time(str, fmt)` with a foldable NULL `fmt` now has type `time(6)`
instead of STRING, consistent with all other forms of `to_time`. The returned
value (NULL) is unchanged. This is a bug fix: the wrong type has been present
since `to_time` was introduced in 4.1.0.
### How was this patch tested?
Added a `TimeExpressionsSuite` case asserting `ToTime(...).dataType ===
TimeType()` for both an untyped and a STRING-typed foldable NULL format; it
fails on the unfixed tree with `StringType did not equal TimeType(6)`. The same
case also asserts the value is still NULL via `checkEvaluation`. Also added two
golden queries in `time.sql` whose schema now locks in `time(6)`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Opus 4.8)
Closes #57486 from LuciferYang/SPARK-58296-totime-null-format.
Authored-by: YangJie <[email protected]>
Signed-off-by: yangjie01 <[email protected]>
(cherry picked from commit 44d1160ae32d1bba04b2da0c36524309a263573b)
Signed-off-by: yangjie01 <[email protected]>
---
.../spark/sql/catalyst/expressions/timeExpressions.scala | 2 +-
.../sql/catalyst/expressions/TimeExpressionsSuite.scala | 14 ++++++++++++++
.../resources/sql-tests/analyzer-results/time.sql.out | 14 ++++++++++++++
sql/core/src/test/resources/sql-tests/inputs/time.sql | 4 ++++
.../src/test/resources/sql-tests/results/time.sql.out | 16 ++++++++++++++++
5 files changed, 49 insertions(+), 1 deletion(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala
index 6469a70349a7..31d87a273e3f 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala
@@ -96,7 +96,7 @@ case class ToTime(str: Expression, format: Option[Expression])
case Some(expr) if expr.foldable =>
Option(expr.eval())
.map(f => invokeParser(Some(f.toString), Seq(str)))
- .getOrElse(Literal(null, expr.dataType))
+ .getOrElse(Literal(null, TimeType()))
case _ => invokeParser()
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala
index a806e2c9419c..5d9dbdf89b7d 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala
@@ -59,6 +59,20 @@ class TimeExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
parameters = Map("input" -> "'100:50'", "format" -> "'mm:HH'"))
}
+ test("SPARK-58296: to_time reports TimeType even when a foldable format is
NULL") {
+ // A foldable format that evaluates to NULL must not change the output
type: to_time
+ // still produces a TIME, matching the non-null and no-format branches.
Before the fix
+ // the replacement fell back to the format argument's own type
(STRING/NULL).
+ assert(new ToTime(Literal("00:00:00"), Literal.create(null,
StringType)).dataType ===
+ TimeType())
+ assert(new ToTime(Literal("00:00:00"), Literal.create(null)).dataType ===
TimeType())
+ // The value is still NULL; assert type and value together for this exact
case.
+ checkEvaluation(new ToTime(Literal("00:00:00"), Literal.create(null,
StringType)), null)
+ // Sanity: the branches that were already correct.
+ assert(new ToTime(Literal("00:00:00")).dataType === TimeType())
+ assert(new ToTime(Literal("00:00:00"), Literal("HH:mm:ss")).dataType ===
TimeType())
+ }
+
test("HourExpressionBuilder") {
// Empty expressions list
checkError(
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out
index 8c5c55bfa0a0..616df51b1091 100644
--- a/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out
+++ b/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out
@@ -48,6 +48,20 @@ Project [to_time(time_str#x, Some(fmt_str#x)) AS
to_time(time_str, fmt_str)#x]
+- OneRowRelation
+-- !query
+select typeof(to_time('00:12:00', null))
+-- !query analysis
+Project [typeof(to_time(00:12:00, Some(null))) AS typeof(to_time(00:12:00,
NULL))#x]
++- OneRowRelation
+
+
+-- !query
+select typeof(to_time('00:12:00', cast(null as string)))
+-- !query analysis
+Project [typeof(to_time(00:12:00, Some(cast(null as string)))) AS
typeof(to_time(00:12:00, CAST(NULL AS STRING)))#x]
++- OneRowRelation
+
+
-- !query
select to_time("11", "HH")
-- !query analysis
diff --git a/sql/core/src/test/resources/sql-tests/inputs/time.sql
b/sql/core/src/test/resources/sql-tests/inputs/time.sql
index f81944716881..1ee7b108f042 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/time.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/time.sql
@@ -11,6 +11,10 @@ select time '16:39:45\t';
select to_time(null), to_time('01:02:03'), to_time('23-59-59.999999',
'HH-mm-ss.SSSSSS');
select to_time(time_str, fmt_str) from time_view;
+-- SPARK-58296: a NULL format must not change the result type of `to_time`
(stays TIME).
+select typeof(to_time('00:12:00', null));
+select typeof(to_time('00:12:00', cast(null as string)));
+
-- missing fields in `to_time`
select to_time("11", "HH");
-- invalid: there is no 13 hours
diff --git a/sql/core/src/test/resources/sql-tests/results/time.sql.out
b/sql/core/src/test/resources/sql-tests/results/time.sql.out
index 5767be09ece5..95fd3ded6caa 100644
--- a/sql/core/src/test/resources/sql-tests/results/time.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/time.sql.out
@@ -47,6 +47,22 @@ struct<to_time(time_str, fmt_str):time(6)>
11:53:26.038344
+-- !query
+select typeof(to_time('00:12:00', null))
+-- !query schema
+struct<typeof(to_time(00:12:00, NULL)):string>
+-- !query output
+time(6)
+
+
+-- !query
+select typeof(to_time('00:12:00', cast(null as string)))
+-- !query schema
+struct<typeof(to_time(00:12:00, CAST(NULL AS STRING))):string>
+-- !query output
+time(6)
+
+
-- !query
select to_time("11", "HH")
-- !query schema
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]