This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c07d9be47060 [SPARK-57816][SQL] Support nanosecond-precision
timestamps in date_format, to_char and to_varchar
c07d9be47060 is described below
commit c07d9be4706083d46658bbfe2a3dcfb26123af38
Author: Stevo Mitric <[email protected]>
AuthorDate: Fri Jul 17 17:26:31 2026 +0800
[SPARK-57816][SQL] Support nanosecond-precision timestamps in date_format,
to_char and to_varchar
### What changes were proposed in this pull request?
Extend `DateFormatClass` (which backs `date_format`, `to_char` and
`to_varchar`) to accept nanosecond-precision timestamps
(`TIMESTAMP_NTZ(p)`/`TIMESTAMP_LTZ(p)`, p in [7, 9]). Both eval and codegen
render via the nanosecond-aware `TimestampFormatter`: NTZ renders its wall
clock zone-independently, LTZ renders in the session zone, and sub-precision
digits floor to zeros. Also guards `SimplifyDateTimeConversions` from rewriting
a nanosecond child, whose round-trip through microsecond ` [...]
### Why are the changes needed?
Nanosecond input was narrowed to microseconds, so the sub-microsecond
digits never reached the formatter.
### Does this PR introduce _any_ user-facing change?
Yes. `date_format`/`to_char`/`to_varchar` now render nanosecond timestamps
at full precision (preview `spark.sql.timestampNanosTypes.enabled`).
### How was this patch tested?
`DateExpressionsSuite`, `SimplifyDateTimeConversionsSuite`,
`TimestampNanosFunctionsSuiteBase`, and `timestamp-{ltz,ntz}-nanos.sql` golden
files.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code
Closes #57225 from stevomitric/stevomitric/spark-57816-nanos.
Authored-by: Stevo Mitric <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../catalyst/expressions/datetimeExpressions.scala | 94 ++++++++++++++++-
.../spark/sql/catalyst/optimizer/expressions.scala | 3 +-
.../expressions/DateExpressionsSuite.scala | 64 +++++++++++
.../SimplifyDateTimeConversionsSuite.scala | 27 +++++
.../analyzer-results/timestamp-ltz-nanos.sql.out | 63 +++++++++++
.../analyzer-results/timestamp-ntz-nanos.sql.out | 56 ++++++++++
.../sql-tests/inputs/timestamp-ltz-nanos.sql | 19 ++++
.../sql-tests/inputs/timestamp-ntz-nanos.sql | 15 +++
.../sql-tests/results/timestamp-ltz-nanos.sql.out | 72 +++++++++++++
.../sql-tests/results/timestamp-ntz-nanos.sql.out | 64 +++++++++++
.../sql/TimestampNanosFunctionsSuiteBase.scala | 117 ++++++++++++++++++++-
11 files changed, 591 insertions(+), 3 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 5e7057ae5165..b8d6933541ef 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -1322,7 +1322,7 @@ case class DateFormatClass(left: Expression, right:
Expression, timeZoneId: Opti
def this(left: Expression, right: Expression) = this(left, right, None)
override def inputTypes: Seq[AbstractDataType] =
- Seq(TypeCollection(TimestampType, AnyTimeType),
+ Seq(TypeCollection(TimestampType, AnyTimeType, AnyTimestampNanoType),
StringTypeWithCollation(supportsTrimCollation = true))
override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
@@ -1335,6 +1335,22 @@ case class DateFormatClass(left: Expression, right:
Expression, timeZoneId: Opti
TimeFormatter(format.toString, isParsing = false))
DateFormatClass.formatTimeWithError(
tf, timestamp.asInstanceOf[Long], prettyName, format.toString)
+ case t: TimestampNTZNanosType =>
+ // NTZ is zone-independent: render the UTC-grid wall clock, ignoring
the session zone.
+ val formatter =
formatterOption.getOrElse(getFormatter(format.toString))
+ DateFormatClass.formatNanosWithError(
+ formatter, timestamp.asInstanceOf[TimestampNanosVal], t.precision,
+ isNTZ = true, prettyName, format.toString)
+ case t: TimestampLTZNanosType =>
+ // LTZ renders in the formatter's session zone.
+ val formatter =
formatterOption.getOrElse(getFormatter(format.toString))
+ DateFormatClass.formatNanosWithError(
+ formatter, timestamp.asInstanceOf[TimestampNanosVal], t.precision,
+ isNTZ = false, prettyName, format.toString)
+ case other: AnyTimestampNanoType =>
+ // AnyTimestampNanoType is not sealed; a future subtype would
otherwise fall through to
+ // the micros branch below and fail with a cryptic ClassCastException.
Mirror doGenCode.
+ throw SparkException.internalError(s"Unexpected nanosecond timestamp
type: $other")
case _ =>
val formatter =
formatterOption.getOrElse(getFormatter(format.toString))
UTF8String.fromString(formatter.format(timestamp.asInstanceOf[Long]))
@@ -1373,6 +1389,50 @@ case class DateFormatClass(left: Expression, right:
Expression, timeZoneId: Opti
|$format.toString()))""".stripMargin.replaceAll("\n", "")
})
}
+ case t @ (_: TimestampNTZNanosType | _: TimestampLTZNanosType) =>
+ // Object carrier: `$timestamp` is the boxed `TimestampNanosVal`
reference (no unbox).
+ // NTZ renders zone-independently; LTZ renders in the formatter's
session zone. Both route
+ // through `DateFormatClass.formatNanosWithError`, which maps an
invalid pattern (e.g. a
+ // zone token over the zone-less NTZ value) to
INVALID_PARAMETER_VALUE.PATTERN. Mirrors the
+ // TIME branch's `formatTimeWithError`.
+ val (isNTZ, precision) = t match {
+ case ntz: TimestampNTZNanosType => (true, ntz.precision)
+ case ltz: TimestampLTZNanosType => (false, ltz.precision)
+ // Compiler exhaustiveness only: `t` is statically DataType, so
scalac cannot see that
+ // the outer case restricts it to the two concrete types. A genuine
future subtype is
+ // caught by the outer `case other: AnyTimestampNanoType` guard
before reaching here.
+ case other => throw SparkException.internalError(
+ s"Unexpected nanosecond timestamp type: $other")
+ }
+ val dfc = DateFormatClass.getClass.getName.stripSuffix("$")
+ val funcName = ctx.addReferenceObj("funcName", prettyName)
+ formatterOption.map { tf =>
+ val timestampFormatter = ctx.addReferenceObj("timestampFormatter",
tf)
+ val fmtStr = ctx.addReferenceObj("fmtStr", right.eval().toString)
+ defineCodeGen(ctx, ev, (timestamp, _) => {
+ s"""$dfc.formatNanosWithError(
+ | $timestampFormatter, $timestamp, $precision, $isNTZ,
$funcName, $fmtStr)"""
+ .stripMargin
+ })
+ }.getOrElse {
+ val tf = TimestampFormatter.getClass.getName.stripSuffix("$")
+ val ldf = LegacyDateFormats.getClass.getName.stripSuffix("$")
+ val zid = ctx.addReferenceObj("zoneId", zoneId,
classOf[ZoneId].getName)
+ defineCodeGen(ctx, ev, (timestamp, format) => {
+ s"""|$dfc.formatNanosWithError(
+ | $tf$$.MODULE$$.apply(
+ | $format.toString(),
+ | $zid,
+ | $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(),
+ | false),
+ | $timestamp, $precision, $isNTZ, $funcName,
$format.toString())""".stripMargin
+ })
+ }
+ case other: AnyTimestampNanoType =>
+ // AnyTimestampNanoType is not sealed; a future subtype would
otherwise fall through to
+ // the micros branch below and emit `format($timestamp)`, treating the
boxed
+ // TimestampNanosVal as a Long. Mirror nullSafeEval's outer guard.
+ throw SparkException.internalError(s"Unexpected nanosecond timestamp
type: $other")
case _ =>
formatterOption.map { tf =>
val timestampFormatter = ctx.addReferenceObj("timestampFormatter",
tf)
@@ -1428,6 +1488,38 @@ object DateFormatClass {
cause = e)
}
}
+
+ /**
+ * Formats a nanosecond-precision timestamp value, mapping an invalid
pattern to a Spark error.
+ * Used by both eval and codegen. NTZ renders zone-independently
(`formatWithoutTimeZoneNanos`);
+ * LTZ renders in the formatter's session zone (`formatNanos`). A pattern
the value cannot satisfy
+ * -- most commonly a zone token (`z`/`Z`/`X`/`O`/`VV`) over the zone-less
NTZ value, which raises
+ * `java.time.DateTimeException` from the underlying `LocalDateTime.format`
-- is surfaced as
+ * INVALID_PARAMETER_VALUE.PATTERN, mirroring [[formatTimeWithError]]
instead of leaking the raw
+ * java.time exception.
+ */
+ def formatNanosWithError(
+ tf: TimestampFormatter,
+ v: TimestampNanosVal,
+ precision: Int,
+ isNTZ: Boolean,
+ funcName: String,
+ pattern: String): UTF8String = {
+ try {
+ val formatted =
+ if (isNTZ) tf.formatWithoutTimeZoneNanos(v, precision) else
tf.formatNanos(v, precision)
+ UTF8String.fromString(formatted)
+ } catch {
+ case e: java.time.DateTimeException =>
+ throw new SparkRuntimeException(
+ errorClass = "INVALID_PARAMETER_VALUE.PATTERN",
+ messageParameters = Map(
+ "parameter" -> toSQLId("format"),
+ "functionName" -> toSQLId(funcName),
+ "value" -> toSQLValue(pattern, StringType)),
+ cause = e)
+ }
+ }
}
/**
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
index de847dddc935..13fa3b09c23c 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -1217,7 +1217,8 @@ object SimplifyDateTimeConversions extends
Rule[LogicalPlan] {
timeZoneId3)
if pattern.semanticEquals(pattern2) &&
pattern.semanticEquals(pattern3)
&& timeZoneId == timeZoneId2 && timeZoneId == timeZoneId3
- && !child.dataType.isInstanceOf[TimeType] =>
+ && !child.dataType.isInstanceOf[TimeType]
+ && !child.dataType.isInstanceOf[AnyTimestampNanoType] =>
e
// Remove a timestamp to string conversion followed by a string to
timestamp conversions if
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 690256156069..69908974367d 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -360,6 +360,70 @@ class DateExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper {
)
}
+ test("SPARK-57816: DateFormat over nanosecond-precision timestamps") {
+ import org.apache.spark.sql.catalyst.util.TimestampNanosTestUtils._
+ val ntz = localDateTimeToNanosVal(timestampNTZ(2020, 1, 1, 13, 24, 35,
123456789))
+ // The 9-`S` pattern is a fixed-width fraction field (date_format uses
isParsing = false), so
+ // it always emits 9 digits; truncating to `p` zeros the low digits rather
than dropping them.
+ Seq(
+ 9 -> "2020-01-01 13:24:35.123456789",
+ 8 -> "2020-01-01 13:24:35.123456780",
+ 7 -> "2020-01-01 13:24:35.123456700").foreach { case (p, expected) =>
+ checkEvaluation(
+ DateFormatClass(Literal.create(ntz, TimestampNTZNanosType(p)),
+ Literal("yyyy-MM-dd HH:mm:ss.SSSSSSSSS"), UTC_OPT),
+ expected)
+ }
+ // LTZ renders in the session zone.
+ val ltz =
instantToNanosVal(Instant.parse("2020-01-01T21:24:35.987654321Z"))
+ checkEvaluation(
+ DateFormatClass(Literal.create(ltz, TimestampLTZNanosType(9)),
+ Literal("yyyy-MM-dd HH:mm:ss.SSSSSSSSS"),
Option("America/Los_Angeles")),
+ "2020-01-01 13:24:35.987654321")
+ // A non-foldable format takes the per-row-formatter path (formatterOption
is None) in both
+ // eval and codegen; checkEvaluation exercises both. The result matches
the foldable case.
+ checkEvaluation(
+ DateFormatClass(Literal.create(ntz, TimestampNTZNanosType(9)),
+ NonFoldableLiteral.create("yyyy-MM-dd HH:mm:ss.SSSSSSSSS",
StringType), UTC_OPT),
+ "2020-01-01 13:24:35.123456789")
+ checkEvaluation(
+ DateFormatClass(Literal.create(ltz, TimestampLTZNanosType(9)),
+ NonFoldableLiteral.create("yyyy-MM-dd HH:mm:ss.SSSSSSSSS", StringType),
+ Option("America/Los_Angeles")),
+ "2020-01-01 13:24:35.987654321")
+ // NULL handling.
+ checkEvaluation(
+ DateFormatClass(Literal.create(null, TimestampNTZNanosType(9)),
+ Literal("yyyy-MM-dd HH:mm:ss.SSSSSSSSS"), UTC_OPT), null)
+
+ // Pre-epoch value combined with sub-precision truncation: epochMicros is
negative but the
+ // sub-micro digits floor toward the precision step independently, so p=7
zeros the low 2
+ // digits of a 1960 timestamp exactly like a post-epoch one.
+ val preEpochNtz = localDateTimeToNanosVal(timestampNTZ(1960, 1, 1, 13, 24,
35, 123456789))
+ Seq(
+ 9 -> "1960-01-01 13:24:35.123456789",
+ 7 -> "1960-01-01 13:24:35.123456700").foreach { case (p, expected) =>
+ checkEvaluation(
+ DateFormatClass(Literal.create(preEpochNtz, TimestampNTZNanosType(p)),
+ Literal("yyyy-MM-dd HH:mm:ss.SSSSSSSSS"), UTC_OPT),
+ expected)
+ }
+
+ // A zone-token pattern (here `z`) cannot render the zone-less NTZ wall
clock: the underlying
+ // LocalDateTime.format raises java.time.DateTimeException, which surfaces
as a clean
+ // INVALID_PARAMETER_VALUE.PATTERN Spark error (mirroring the TIME path)
rather than leaking the
+ // raw java.time exception. checkErrorInExpression exercises both eval and
codegen.
+ checkErrorInExpression[SparkRuntimeException](
+ DateFormatClass(Literal.create(ntz, TimestampNTZNanosType(9)),
+ Literal("yyyy-MM-dd HH:mm:ss z"), UTC_OPT),
+ condition = "INVALID_PARAMETER_VALUE.PATTERN",
+ parameters = Map(
+ "parameter" -> "`format`",
+ "functionName" -> "`date_format`",
+ "value" -> "'yyyy-MM-dd HH:mm:ss z'")
+ )
+ }
+
test("Hour") {
assert(Hour(Literal.create(null, DateType), UTC_OPT).resolved === false)
assert(Hour(Literal(ts), UTC_OPT).resolved)
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyDateTimeConversionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyDateTimeConversionsSuite.scala
index e67d8f2dc7f8..32fa999b65c2 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyDateTimeConversionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyDateTimeConversionsSuite.scala
@@ -91,4 +91,31 @@ class SimplifyDateTimeConversionsSuite extends PlanTest {
// Should NOT be simplified - optimized plan should equal original (no
rewrite)
comparePlans(optimized, originalQuery)
}
+
+ test("SPARK-57816: SimplifyDateTimeConversions skips nanosecond timestamp
child") {
+ val pattern = "yyyy-MM-dd HH:mm:ss.SSSSSSSSS"
+ Seq(TimestampNTZNanosType(9), TimestampLTZNanosType(9)).foreach {
nanosType =>
+ val nanosAttr = AttributeReference("t", nanosType)()
+ val nanosRelation = LocalRelation(nanosAttr)
+
+ val df = DateFormatClass(nanosAttr, pattern)
+
+ // date_format(to_timestamp(date_format(nanos_col, p), p), p) should NOT
simplify because
+ // the round trip through micro TimestampType truncates sub-microsecond
precision.
+ val originalQuery = nanosRelation
+ .select(
+ DateFormatClass(
+ GetTimestamp(
+ df,
+ pattern,
+ TimestampType),
+ pattern) as "c1")
+ .analyze
+
+ val optimized = Optimize.execute(originalQuery)
+
+ // Should NOT be simplified - optimized plan should equal original (no
rewrite)
+ comparePlans(optimized, originalQuery)
+ }
+ }
}
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out
index 454eeddf2f23..3d38acc777e6 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out
@@ -1036,6 +1036,69 @@ Sort [v#x ASC NULLS FIRST], true
+- OneRowRelation
+-- !query
+SELECT date_format(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(2020-01-01 13:24:35.123456789, yyyy-MM-dd
HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS date_format(TIMESTAMP_LTZ
'2020-01-01 13:24:35.123456789', yyyy-MM-dd HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT date_format('2020-01-01 13:24:35.123456789' :: timestamp_ltz(8),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(cast(2020-01-01 13:24:35.123456789 as timestamp_ltz(8)),
yyyy-MM-dd HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS
date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_LTZ(8)), yyyy-MM-dd
HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT date_format('2020-01-01 13:24:35.123456789' :: timestamp_ltz(7),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(cast(2020-01-01 13:24:35.123456789 as timestamp_ltz(7)),
yyyy-MM-dd HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS
date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_LTZ(7)), yyyy-MM-dd
HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT date_format(TIMESTAMP_LTZ '2020-01-01 21:24:35.123456789 UTC',
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(2020-01-01 13:24:35.123456789, yyyy-MM-dd
HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS date_format(TIMESTAMP_LTZ
'2020-01-01 13:24:35.123456789', yyyy-MM-dd HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT to_char(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(2020-01-01 13:24:35.123456789, yyyy-MM-dd
HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS date_format(TIMESTAMP_LTZ
'2020-01-01 13:24:35.123456789', yyyy-MM-dd HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT to_char('2020-01-01 13:24:35.123456789' :: timestamp_ltz(7),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(cast(2020-01-01 13:24:35.123456789 as timestamp_ltz(7)),
yyyy-MM-dd HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS
date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_LTZ(7)), yyyy-MM-dd
HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT to_varchar('2020-01-01 13:24:35.123456789' :: timestamp_ltz(8),
'HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(cast(2020-01-01 13:24:35.123456789 as timestamp_ltz(8)),
HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS date_format(CAST(2020-01-01
13:24:35.123456789 AS TIMESTAMP_LTZ(8)), HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT date_format(TIMESTAMP_LTZ '1960-01-01 13:24:35.123456789 UTC',
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(1960-01-01 05:24:35.123456789, yyyy-MM-dd
HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS date_format(TIMESTAMP_LTZ
'1960-01-01 05:24:35.123456789', yyyy-MM-dd HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT date_format(NULL :: timestamp_ltz(9), 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(cast(null as timestamp_ltz(9)), yyyy-MM-dd
HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS date_format(CAST(NULL AS
TIMESTAMP_LTZ(9)), yyyy-MM-dd HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
-- !query
SELECT date_trunc('SECOND', TIMESTAMP_LTZ '2020-01-01 12:34:56.123456789 UTC')
-- !query analysis
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out
index 879c084a7496..ff49a5870e29 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out
@@ -870,6 +870,62 @@ Sort [v#x ASC NULLS FIRST], true
+- OneRowRelation
+-- !query
+SELECT date_format(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(2020-01-01 13:24:35.123456789, yyyy-MM-dd
HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS date_format(TIMESTAMP_NTZ
'2020-01-01 13:24:35.123456789', yyyy-MM-dd HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT date_format('2020-01-01 13:24:35.123456789' :: timestamp_ntz(8),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(cast(2020-01-01 13:24:35.123456789 as timestamp_ntz(8)),
yyyy-MM-dd HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS
date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_NTZ(8)), yyyy-MM-dd
HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT date_format('2020-01-01 13:24:35.123456789' :: timestamp_ntz(7),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(cast(2020-01-01 13:24:35.123456789 as timestamp_ntz(7)),
yyyy-MM-dd HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS
date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_NTZ(7)), yyyy-MM-dd
HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT to_char(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(2020-01-01 13:24:35.123456789, yyyy-MM-dd
HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS date_format(TIMESTAMP_NTZ
'2020-01-01 13:24:35.123456789', yyyy-MM-dd HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT to_char('2020-01-01 13:24:35.123456789' :: timestamp_ntz(7),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(cast(2020-01-01 13:24:35.123456789 as timestamp_ntz(7)),
yyyy-MM-dd HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS
date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_NTZ(7)), yyyy-MM-dd
HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT to_varchar('2020-01-01 13:24:35.123456789' :: timestamp_ntz(8),
'HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(cast(2020-01-01 13:24:35.123456789 as timestamp_ntz(8)),
HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS date_format(CAST(2020-01-01
13:24:35.123456789 AS TIMESTAMP_NTZ(8)), HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT date_format(TIMESTAMP_NTZ '1960-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(1960-01-01 13:24:35.123456789, yyyy-MM-dd
HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS date_format(TIMESTAMP_NTZ
'1960-01-01 13:24:35.123456789', yyyy-MM-dd HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
+-- !query
+SELECT date_format(NULL :: timestamp_ntz(9), 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query analysis
+Project [date_format(cast(null as timestamp_ntz(9)), yyyy-MM-dd
HH:mm:ss.SSSSSSSSS, Some(America/Los_Angeles)) AS date_format(CAST(NULL AS
TIMESTAMP_NTZ(9)), yyyy-MM-dd HH:mm:ss.SSSSSSSSS)#x]
++- OneRowRelation
+
+
-- !query
SELECT date_trunc('SECOND', TIMESTAMP_NTZ '2020-01-01 12:34:56.123456789')
-- !query analysis
diff --git
a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql
b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql
index a6bed71cc1ac..427f89c96666 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql
@@ -329,6 +329,25 @@ SELECT v, lead(v) OVER (ORDER BY v) AS next_v FROM (
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_LTZ '2020-01-01 00:00:00.000000500') ORDER BY v;
+-- SPARK-57816: date_format / to_char / to_varchar over nanosecond-precision
values. The pattern's
+-- fractional-second placeholders render up to nanosecond digits; a 9-`S`
field is fixed width, so
+-- digits below the type's precision floor to zeros rather than being dropped.
LTZ renders in the
+-- session time zone (America/Los_Angeles, UTC-08:00), so a bare literal
round-trips while an
+-- explicit-zone literal shifts the rendered wall clock. to_char / to_varchar
route through the
+-- same code path.
+SELECT date_format(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS');
+SELECT date_format('2020-01-01 13:24:35.123456789' :: timestamp_ltz(8),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS');
+SELECT date_format('2020-01-01 13:24:35.123456789' :: timestamp_ltz(7),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS');
+-- An explicit-zone literal (UTC) is shifted into the session zone before
rendering.
+SELECT date_format(TIMESTAMP_LTZ '2020-01-01 21:24:35.123456789 UTC',
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS');
+SELECT to_char(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS');
+SELECT to_char('2020-01-01 13:24:35.123456789' :: timestamp_ltz(7),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS');
+SELECT to_varchar('2020-01-01 13:24:35.123456789' :: timestamp_ltz(8),
'HH:mm:ss.SSSSSSSSS');
+-- Pre-epoch value exercises the negative-epoch path.
+SELECT date_format(TIMESTAMP_LTZ '1960-01-01 13:24:35.123456789 UTC',
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS');
+-- NULL nanosecond timestamp.
+SELECT date_format(NULL :: timestamp_ltz(9), 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS');
+
-- SPARK-57821: date_trunc keeps the nanosecond type/family and zeroes the
whole fraction (including
-- the sub-microsecond digits); MICROSECOND keeps epochMicros and only drops
nanosWithinMicro. LTZ
-- truncates in the session time zone (America/Los_Angeles, UTC-08:00), so DAY
over an early-hours
diff --git
a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql
b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql
index aa173fb72620..384acca073be 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql
@@ -269,6 +269,21 @@ SELECT v, lead(v) OVER (ORDER BY v) AS next_v FROM (
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000100'
UNION ALL SELECT TIMESTAMP_NTZ '2020-01-01 00:00:00.000000500') ORDER BY v;
+-- SPARK-57816: date_format / to_char / to_varchar over nanosecond-precision
values. The pattern's
+-- fractional-second placeholders render up to nanosecond digits; a 9-`S`
field is fixed width, so
+-- digits below the type's precision floor to zeros rather than being dropped.
NTZ renders its
+-- wall clock zone-independently. to_char / to_varchar route through the same
code path.
+SELECT date_format(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS');
+SELECT date_format('2020-01-01 13:24:35.123456789' :: timestamp_ntz(8),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS');
+SELECT date_format('2020-01-01 13:24:35.123456789' :: timestamp_ntz(7),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS');
+SELECT to_char(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS');
+SELECT to_char('2020-01-01 13:24:35.123456789' :: timestamp_ntz(7),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS');
+SELECT to_varchar('2020-01-01 13:24:35.123456789' :: timestamp_ntz(8),
'HH:mm:ss.SSSSSSSSS');
+-- Pre-epoch value exercises the negative-epoch path.
+SELECT date_format(TIMESTAMP_NTZ '1960-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS');
+-- NULL nanosecond timestamp.
+SELECT date_format(NULL :: timestamp_ntz(9), 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS');
+
-- SPARK-57821: date_trunc keeps the nanosecond type/family and zeroes the
whole fraction (including
-- the sub-microsecond digits); MICROSECOND keeps epochMicros and only drops
nanosWithinMicro. NTZ
-- is zone-independent, so DAY/HOUR read the wall clock and never shift.
diff --git
a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out
b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out
index 8aa3586481e7..f1f3b7f61022 100644
--- a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out
@@ -1127,6 +1127,78 @@ struct<v:timestamp_ltz(9),next_v:timestamp_ltz(9)>
2020-01-01 00:00:00.0000009 NULL
+-- !query
+SELECT date_format(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789', yyyy-MM-dd
HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+2020-01-01 13:24:35.123456789
+
+
+-- !query
+SELECT date_format('2020-01-01 13:24:35.123456789' :: timestamp_ltz(8),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_LTZ(8)),
yyyy-MM-dd HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+2020-01-01 13:24:35.123456780
+
+
+-- !query
+SELECT date_format('2020-01-01 13:24:35.123456789' :: timestamp_ltz(7),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_LTZ(7)),
yyyy-MM-dd HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+2020-01-01 13:24:35.123456700
+
+
+-- !query
+SELECT date_format(TIMESTAMP_LTZ '2020-01-01 21:24:35.123456789 UTC',
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789', yyyy-MM-dd
HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+2020-01-01 13:24:35.123456789
+
+
+-- !query
+SELECT to_char(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789', yyyy-MM-dd
HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+2020-01-01 13:24:35.123456789
+
+
+-- !query
+SELECT to_char('2020-01-01 13:24:35.123456789' :: timestamp_ltz(7),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_LTZ(7)),
yyyy-MM-dd HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+2020-01-01 13:24:35.123456700
+
+
+-- !query
+SELECT to_varchar('2020-01-01 13:24:35.123456789' :: timestamp_ltz(8),
'HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_LTZ(8)),
HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+13:24:35.123456780
+
+
+-- !query
+SELECT date_format(TIMESTAMP_LTZ '1960-01-01 13:24:35.123456789 UTC',
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(TIMESTAMP_LTZ '1960-01-01 05:24:35.123456789', yyyy-MM-dd
HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+1960-01-01 05:24:35.123456789
+
+
+-- !query
+SELECT date_format(NULL :: timestamp_ltz(9), 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(CAST(NULL AS TIMESTAMP_LTZ(9)), yyyy-MM-dd
HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+NULL
+
+
-- !query
SELECT date_trunc('SECOND', TIMESTAMP_LTZ '2020-01-01 12:34:56.123456789 UTC')
-- !query schema
diff --git
a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out
b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out
index acb2fe195115..dbb3a77295ee 100644
--- a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out
@@ -924,6 +924,70 @@ struct<v:timestamp_ntz(9),next_v:timestamp_ntz(9)>
2020-01-01 00:00:00.0000009 NULL
+-- !query
+SELECT date_format(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789', yyyy-MM-dd
HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+2020-01-01 13:24:35.123456789
+
+
+-- !query
+SELECT date_format('2020-01-01 13:24:35.123456789' :: timestamp_ntz(8),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_NTZ(8)),
yyyy-MM-dd HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+2020-01-01 13:24:35.123456780
+
+
+-- !query
+SELECT date_format('2020-01-01 13:24:35.123456789' :: timestamp_ntz(7),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_NTZ(7)),
yyyy-MM-dd HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+2020-01-01 13:24:35.123456700
+
+
+-- !query
+SELECT to_char(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789', yyyy-MM-dd
HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+2020-01-01 13:24:35.123456789
+
+
+-- !query
+SELECT to_char('2020-01-01 13:24:35.123456789' :: timestamp_ntz(7),
'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_NTZ(7)),
yyyy-MM-dd HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+2020-01-01 13:24:35.123456700
+
+
+-- !query
+SELECT to_varchar('2020-01-01 13:24:35.123456789' :: timestamp_ntz(8),
'HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(CAST(2020-01-01 13:24:35.123456789 AS TIMESTAMP_NTZ(8)),
HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+13:24:35.123456780
+
+
+-- !query
+SELECT date_format(TIMESTAMP_NTZ '1960-01-01 13:24:35.123456789', 'yyyy-MM-dd
HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(TIMESTAMP_NTZ '1960-01-01 13:24:35.123456789', yyyy-MM-dd
HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+1960-01-01 13:24:35.123456789
+
+
+-- !query
+SELECT date_format(NULL :: timestamp_ntz(9), 'yyyy-MM-dd HH:mm:ss.SSSSSSSSS')
+-- !query schema
+struct<date_format(CAST(NULL AS TIMESTAMP_NTZ(9)), yyyy-MM-dd
HH:mm:ss.SSSSSSSSS):string>
+-- !query output
+NULL
+
+
-- !query
SELECT date_trunc('SECOND', TIMESTAMP_NTZ '2020-01-01 12:34:56.123456789')
-- !query schema
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala
b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala
index ee350f5c5e2d..14e346753d8e 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql
import java.time.{Instant, LocalDateTime}
-import org.apache.spark.SparkConf
+import org.apache.spark.{SparkConf, SparkRuntimeException,
SparkUnsupportedOperationException}
import org.apache.spark.sql.functions._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
@@ -666,6 +666,121 @@ abstract class TimestampNanosFunctionsSuiteBase extends
SharedSparkSession {
}
}
}
+
+ test("SPARK-57816: date_format / to_char / to_varchar over
nanosecond-precision timestamps") {
+ // The 9-`S` pattern is a fixed-width fraction field, so it always emits 9
digits; truncating to
+ // precision `p` zeros the low digits (floor); it does not drop them. The
session zone is
+ // America/Los_Angeles: TIMESTAMP_NTZ renders its wall clock
zone-independently, while the
+ // TIMESTAMP_LTZ instant (21:24:35 UTC) shifts to 13:24:35 in the session
zone (UTC-8 in Jan).
+ val fmt = "yyyy-MM-dd HH:mm:ss.SSSSSSSSS"
+ val ntzStr = "2020-01-01T13:24:35.123456789"
+ val ltzStr = "2020-01-01T21:24:35.987654321Z"
+ Seq(
+ 9 -> ("2020-01-01 13:24:35.123456789", "2020-01-01 13:24:35.987654321"),
+ 8 -> ("2020-01-01 13:24:35.123456780", "2020-01-01 13:24:35.987654320"),
+ 7 -> ("2020-01-01 13:24:35.123456700", "2020-01-01 13:24:35.987654300")
+ ).foreach { case (p, (ntzExpected, ltzExpected)) =>
+ val ntz = ntzNanos(ntzStr, p)
+ // SQL path: date_format, to_char, to_varchar all render sub-microsecond
digits.
+ checkAnswer(
+ ntz.selectExpr(
+ s"date_format(c, '$fmt')", s"to_char(c, '$fmt')", s"to_varchar(c,
'$fmt')"),
+ Row(ntzExpected, ntzExpected, ntzExpected))
+ // Column API agrees with the SQL path.
+ checkAnswer(
+ ntz.select(date_format(col("c"), fmt), to_char(col("c"), lit(fmt)),
+ to_varchar(col("c"), lit(fmt))),
+ Row(ntzExpected, ntzExpected, ntzExpected))
+
+ val ltz = ltzNanos(ltzStr, p)
+ checkAnswer(
+ ltz.selectExpr(
+ s"date_format(c, '$fmt')", s"to_char(c, '$fmt')", s"to_varchar(c,
'$fmt')"),
+ Row(ltzExpected, ltzExpected, ltzExpected))
+ checkAnswer(
+ ltz.select(date_format(col("c"), fmt), to_char(col("c"), lit(fmt)),
+ to_varchar(col("c"), lit(fmt))),
+ Row(ltzExpected, ltzExpected, ltzExpected))
+ }
+
+ // NULL nanosecond input renders NULL for all three functions.
+ Seq(7, 8, 9).foreach { p =>
+ val ntzNull = spark.createDataFrame(
+ spark.sparkContext.parallelize(Seq(Row(null))),
+ new StructType().add("c", TimestampNTZNanosType(p)))
+ val ltzNull = spark.createDataFrame(
+ spark.sparkContext.parallelize(Seq(Row(null))),
+ new StructType().add("c", TimestampLTZNanosType(p)))
+ checkAnswer(
+ ntzNull.selectExpr(
+ s"date_format(c, '$fmt')", s"to_char(c, '$fmt')", s"to_varchar(c,
'$fmt')"),
+ Row(null, null, null))
+ checkAnswer(
+ ltzNull.selectExpr(
+ s"date_format(c, '$fmt')", s"to_char(c, '$fmt')", s"to_varchar(c,
'$fmt')"),
+ Row(null, null, null))
+ }
+ }
+
+ test("SPARK-57816: date_format / to_char / to_varchar over nanos reject the
LEGACY " +
+ "time parser policy") {
+ // date_format never sets forTimestampNTZ, so under LEGACY it builds a
legacy formatter for
+ // both NTZ and LTZ nanos, whose nanos format methods raise
+ // TIMESTAMP_NANOS_WITH_LEGACY_TIME_PARSER (a micros value would format
fine). Mirrors the
+ // JSON-path LEGACY coverage (SPARK-57456) at the expression level.
+ def rootNanosError(e: Throwable): SparkUnsupportedOperationException = {
+ var cause = e
+ while (cause != null &&
!cause.isInstanceOf[SparkUnsupportedOperationException]) {
+ cause = cause.getCause
+ }
+ assert(cause != null, s"Expected
TIMESTAMP_NANOS_WITH_LEGACY_TIME_PARSER, but got: $e")
+ cause.asInstanceOf[SparkUnsupportedOperationException]
+ }
+ withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "LEGACY") {
+ val fmt = "yyyy-MM-dd HH:mm:ss.SSSSSSSSS"
+ val expectedParameters =
+ Map("config" -> ("\"" + SQLConf.LEGACY_TIME_PARSER_POLICY.key + "\""))
+ val dfs = Seq(
+ ntzNanos("2020-01-01T13:24:35.123456789", 9),
+ ltzNanos("2020-01-01T21:24:35.987654321Z", 9))
+ dfs.foreach { df =>
+ Seq(s"date_format(c, '$fmt')", s"to_char(c, '$fmt')", s"to_varchar(c,
'$fmt')")
+ .foreach { expr =>
+ checkError(
+ exception = rootNanosError(intercept[Exception] {
+ df.selectExpr(expr).collect()
+ }),
+ condition =
"UNSUPPORTED_FEATURE.TIMESTAMP_NANOS_WITH_LEGACY_TIME_PARSER",
+ parameters = expectedParameters)
+ }
+ }
+ }
+ }
+
+ test("SPARK-57816: date_format / to_char / to_varchar over NTZ nanos reject
a zone-token " +
+ "pattern with a clean error") {
+ // A zone token (`z`, `Z`, `X`, `O`, `VV`) has no meaning for the
zone-less TIMESTAMP_NTZ wall
+ // clock, so rendering raises java.time.DateTimeException underneath.
date_format maps it to a
+ // clean INVALID_PARAMETER_VALUE.PATTERN Spark error rather than leaking
the raw java.time
+ // exception. LTZ is zone-aware, so the same pattern renders the session
zone instead of
+ // erroring - hence this is NTZ-only.
+ val ntz = ntzNanos("2020-01-01T13:24:35.123456789", 9)
+ Seq("z", "Z", "X", "O", "VV").foreach { zoneToken =>
+ val fmt = s"yyyy-MM-dd HH:mm:ss $zoneToken"
+ Seq(s"date_format(c, '$fmt')", s"to_char(c, '$fmt')", s"to_varchar(c,
'$fmt')")
+ .foreach { expr =>
+ checkError(
+ exception = intercept[SparkRuntimeException] {
+ ntz.selectExpr(expr).collect()
+ },
+ condition = "INVALID_PARAMETER_VALUE.PATTERN",
+ parameters = Map(
+ "parameter" -> "`format`",
+ "functionName" -> "`date_format`",
+ "value" -> s"'$fmt'"))
+ }
+ }
+ }
}
// Runs the nanosecond timestamp function tests with ANSI mode enabled
explicitly.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
