stevomitric commented on code in PR #56466:
URL: https://github.com/apache/spark/pull/56466#discussion_r3404722350
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala:
##########
@@ -468,6 +468,65 @@ case class SecondWithFraction(child: Expression,
timeZoneId: Option[String] = No
copy(child = newChild)
}
+/**
+ * Returns the second component with the fractional part of a
nanosecond-precision timestamp
+ * (`TIMESTAMP_NTZ(p)` / `TIMESTAMP_LTZ(p)`, `p` in `[7, 9]`) as `DECIMAL(11,
9)`.
+ *
+ * This is the nanosecond counterpart of [[SecondWithFraction]] on the
+ * `EXTRACT(SECOND FROM ...)` / `date_part('SECOND', ...)` path. Unlike the
integer time-of-day
+ * fields, the result depends on the sub-microsecond digits, so the input is
not cast down to
+ * microseconds. The scale is fixed at the maximum nanosecond precision, like
+ * [[SecondsOfTimeWithFraction]] does for TIME values: digits below the type's
precision `p` are
+ * always zero because values are floored to `p` at the type boundary.
+ */
+case class SecondWithFractionNanos(child: Expression, timeZoneId:
Option[String] = None)
+ extends UnaryExpression with TimeZoneAwareExpression {
+
+ def this(child: Expression) = this(child, None)
+
+ override def nullIntolerant: Boolean = true
+
+ // 2 digits for the seconds, and 9 digits for the fractional part with
nanosecond precision.
+ override def dataType: DataType = DecimalType(11, 9)
+
+ override def checkInputDataTypes(): TypeCheckResult = child.dataType match {
+ case _: TimestampNTZNanosType | _: TimestampLTZNanosType =>
TypeCheckSuccess
+ case _ =>
+ DataTypeMismatch(
+ errorSubClass = "UNEXPECTED_INPUT_TYPE",
+ messageParameters = Map(
+ "paramIndex" -> ordinalNumber(0),
+ "requiredType" ->
+ Seq(TimestampNTZNanosType(),
TimestampLTZNanosType()).map(toSQLType).mkString(" or "),
+ "inputSql" -> toSQLExpr(child),
+ "inputType" -> toSQLType(child.dataType)))
+ }
+
+ override def withTimeZone(timeZoneId: String): SecondWithFractionNanos =
+ copy(timeZoneId = Option(timeZoneId))
+
+ // TIMESTAMP_NTZ(p) extracts the wall-clock fields, so it is evaluated in
UTC like
+ // TimestampNTZType in zoneIdForType; TIMESTAMP_LTZ(p) uses the session time
zone.
+ @transient private lazy val zoneIdInEval: ZoneId = child.dataType match {
+ case _: TimestampNTZNanosType => ZoneOffset.UTC
Review Comment:
Added TimestampNTZNanosType to the NTZ→UTC arm in zoneIdForType
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala:
##########
@@ -3341,9 +3400,22 @@ object DatePart {
case "DAYOFWEEK" | "DOW" => DayOfWeek(source)
case "DAYOFWEEK_ISO" | "DOW_ISO" => Add(WeekDay(source), Literal(1))
case "DOY" => DayOfYear(source)
- case "HOUR" | "H" | "HOURS" | "HR" | "HRS" => Hour(source)
- case "MINUTE" | "M" | "MIN" | "MINS" | "MINUTES" => Minute(source)
- case "SECOND" | "S" | "SEC" | "SECONDS" | "SECS" =>
SecondWithFraction(source)
+ case "HOUR" | "H" | "HOURS" | "HR" | "HRS" =>
+ // Nanosecond-precision timestamps are cast down to the matching
microsecond timestamp
+ // type, which is lossless for the integer time-of-day fields
(SPARK-57340); other types
+ // are passed through unchanged.
+ Hour(NanosTimestampCast.castToMicros(source))
+ case "MINUTE" | "M" | "MIN" | "MINS" | "MINUTES" =>
+ Minute(NanosTimestampCast.castToMicros(source))
+ case "SECOND" | "S" | "SEC" | "SECONDS" | "SECS" =>
+ source.dataType match {
+ // EXTRACT(SECOND) exposes the fractional digits, so
nanosecond-precision timestamps
+ // keep their sub-microsecond digits and widen the result to
DECIMAL(11, 9) instead of
+ // casting down to microseconds (SPARK-57340).
+ case _: TimestampNTZNanosType | _: TimestampLTZNanosType =>
+ SecondWithFractionNanos(source)
Review Comment:
added a note
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
