This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new d1828e710251 [SPARK-57815][SQL] Reject nanosecond-precision timestamps
in Hive-serde tables
d1828e710251 is described below
commit d1828e710251d27d7245b5fedf48ccd313ecc585
Author: Stevo Mitric <[email protected]>
AuthorDate: Fri Jul 31 09:11:15 2026 +0000
[SPARK-57815][SQL] Reject nanosecond-precision timestamps in Hive-serde
tables
### What changes were proposed in this pull request?
Handle nanosecond-precision timestamp columns (`TIMESTAMP_LTZ(p)` /
`TIMESTAMP_NTZ(p)`, `p` in [7, 9]) in the Hive-serde (`STORED AS`) write path
by rejecting them explicitly, mirroring how the TIME type is handled
(SPARK-57556):
- `HiveFileFormat.supportDataType` now returns `false` for
`AnyTimestampNanoType` (recursing into nested types), so writing such a column
fails with a clear `UNSUPPORTED_DATA_TYPE_FOR_DATASOURCE` error at
`verifySchema` time instead of silently downgrading to microsecond precision.
- `HiveInspectors.toInspector(dataType)`, `toInspector(expr)` (for
`Literal`), and `toTypeInfo` now throw `UNSUPPORTED_DATATYPE` for
`AnyTimestampNanoType`, since Hive has no nanosecond-precision timestamp type
to map to.
Rejection (rather than full support) is the intentional behavior chosen by
the umbrella: Hive has no nanosecond timestamp type, and the
metastore-compatibility side already treats these types as Hive-incompatible
(SPARK-57831).
### Why are the changes needed?
Sub-task of SPARK-56822. Without this, the Hive ORC serde silently
downgrades nanosecond timestamps to microsecond `TimestampType` on write,
losing precision with no warning. Silent downgrades must be eliminated.
### Does this PR introduce any user-facing change?
Yes. Writing a nanosecond-precision timestamp column through the Hive serde
now fails with a clear error instead of silently losing sub-microsecond
precision.
### How was this patch tested?
New unit test in `HiveInspectorSuite` covering `toInspector`/`toTypeInfo`
for all four nanos type/precision combinations, and a new `InsertSuite` test
that writes to a Hive-serde directory `STORED AS ORC` and `STORED AS PARQUET`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 4.8)
Closes #57635 from stevomitric/stevomitric/spark-57815-nanos.
Authored-by: Stevo Mitric <[email protected]>
Signed-off-by: Uros Bojanic <[email protected]>
---
.../org/apache/spark/sql/hive/HiveInspectors.scala | 9 ++++++
.../spark/sql/hive/execution/HiveFileFormat.scala | 6 +++-
.../apache/spark/sql/hive/HiveInspectorSuite.scala | 20 +++++++++++++
.../org/apache/spark/sql/hive/InsertSuite.scala | 34 ++++++++++++++++++++++
4 files changed, 68 insertions(+), 1 deletion(-)
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 7f76132e2572..662e55dac543 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -967,6 +967,9 @@ private[hive] trait HiveInspectors {
toInspector(sqlType)
// Hive has no TIME type, so it cannot be represented by any Hive object
inspector.
case _: TimeType => throw unsupportedHiveType(dataType)
+ // Hive has no nanosecond-precision timestamp type, so it cannot be
represented by any Hive
+ // object inspector. Reject it instead of silently downgrading to
microsecond precision.
+ case _: AnyTimestampNanoType => throw unsupportedHiveType(dataType)
}
private def unsupportedHiveType(dataType: DataType): AnalysisException = {
@@ -1044,6 +1047,10 @@ private[hive] trait HiveInspectors {
// Hive has no TIME type, so a TIME constant cannot be mapped to a Hive
object inspector.
case Literal(_, dt: TimeType) =>
throw unsupportedHiveType(dt)
+ // Hive has no nanosecond-precision timestamp type, so such a constant
cannot be mapped to a
+ // Hive object inspector.
+ case Literal(_, dt: AnyTimestampNanoType) =>
+ throw unsupportedHiveType(dt)
// We will enumerate all of the possible constant expressions, throw
exception if we missed
case Literal(_, dt) =>
throw SparkException.internalError(s"Hive doesn't support the constant
type [$dt].")
@@ -1297,6 +1304,8 @@ private[hive] trait HiveInspectors {
case _: YearMonthIntervalType => intervalYearMonthTypeInfo
// Hive has no TIME type, so there is no Hive TypeInfo to map it to.
case _: TimeType => throw unsupportedHiveType(dt)
+ // Hive has no nanosecond-precision timestamp type, so there is no Hive
TypeInfo to map it to.
+ case _: AnyTimestampNanoType => throw unsupportedHiveType(dt)
case dt => throw unsupportedHiveType(dt)
}
}
diff --git
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
index acbc72fbf7e0..4545acc7f203 100644
---
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
+++
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
@@ -41,7 +41,7 @@ import
org.apache.spark.sql.execution.datasources.{FileFormat, OutputWriter, Out
import org.apache.spark.sql.hive.{HiveInspectors, HiveTableUtil}
import org.apache.spark.sql.internal.SessionStateHelper
import org.apache.spark.sql.sources.DataSourceRegister
-import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType,
TimeType, UserDefinedType}
+import org.apache.spark.sql.types.{AnyTimestampNanoType, ArrayType, DataType,
MapType, StructType, TimeType, UserDefinedType}
import org.apache.spark.util.SerializableJobConf
/**
@@ -120,6 +120,10 @@ case class HiveFileFormat(fileSinkConf: FileSinkDesc)
// (recursing into nested types) while preserving the default behavior for
all other types.
case _: TimeType => false
+ // Hive has no nanosecond-precision timestamp type. Reject it explicitly
rather than silently
+ // downgrading to microsecond precision (which the Hive serde would
otherwise do).
+ case _: AnyTimestampNanoType => false
+
case st: StructType => st.forall { f => supportDataType(f.dataType) }
case ArrayType(elementType, _) => supportDataType(elementType)
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index b7fb506f07b5..a88dcd0ef56a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -308,4 +308,24 @@ class HiveInspectorSuite extends SparkFunSuite with
HiveInspectors {
condition = "UNSUPPORTED_DATATYPE",
parameters = expectedParams)
}
+
+ test("SPARK-57815: nanosecond timestamp types are unsupported in Hive object
inspectors") {
+ Seq(
+ TimestampNTZNanosType(9), TimestampNTZNanosType(7),
+ TimestampLTZNanosType(9), TimestampLTZNanosType(8)).foreach { nanosType
=>
+ val expectedParams = Map("typeName" -> s"\"${nanosType.sql}\"")
+ checkError(
+ exception = intercept[AnalysisException](toInspector(nanosType)),
+ condition = "UNSUPPORTED_DATATYPE",
+ parameters = expectedParams)
+ checkError(
+ exception =
intercept[AnalysisException](toInspector(Literal.create(null, nanosType))),
+ condition = "UNSUPPORTED_DATATYPE",
+ parameters = expectedParams)
+ checkError(
+ exception = intercept[AnalysisException](nanosType.toTypeInfo),
+ condition = "UNSUPPORTED_DATATYPE",
+ parameters = expectedParams)
+ }
+ }
}
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala
index 7385410e1a71..db1743862e90 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala
@@ -708,6 +708,40 @@ class InsertSuite extends QueryTest with TestHiveSingleton
with BeforeAndAfter {
}
}
+ test("SPARK-57815: nanosecond timestamp is unsupported when writing to a
Hive serde directory") {
+ // Disable native data source conversion so that the write goes through
the Hive serde path
+ // (HiveFileFormat) instead of a native data source that supports
nanosecond timestamps.
+ withSQLConf(
+ HiveUtils.CONVERT_METASTORE_INSERT_DIR.key -> "false",
+ SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true") {
+ Seq("ORC", "PARQUET").foreach { fileFormat =>
+ Seq(
+ "TIMESTAMP_NTZ(9)" -> TimestampNTZNanosType(9),
+ "TIMESTAMP_LTZ(9)" -> TimestampLTZNanosType(9)).foreach { case
(typeStr, dt) =>
+ withTempDir { dir =>
+ // InsertIntoHiveDirCommand wraps the failure in a SparkException,
so assert on the
+ // cause. Rejecting here avoids a silent downgrade to microsecond
precision.
+ val e = intercept[SparkException] {
+ sql(
+ s"""
+ |INSERT OVERWRITE LOCAL DIRECTORY '${dir.toURI.getPath}'
+ |STORED AS $fileFormat
+ |SELECT CAST('2025-01-06 12:30:45.123456789' AS $typeStr)
AS c
+ """.stripMargin)
+ }
+ checkError(
+ exception = e.getCause.asInstanceOf[AnalysisException],
+ condition = "UNSUPPORTED_DATA_TYPE_FOR_DATASOURCE",
+ parameters = Map(
+ "columnName" -> "`c`",
+ "columnType" -> s"\"${dt.sql}\"",
+ "format" -> "Hive"))
+ }
+ }
+ }
+ }
+ }
+
test("insert overwrite to dir from temp table") {
withTempView("test_insert_table") {
spark.range(10).selectExpr("id", "id AS
str").createOrReplaceTempView("test_insert_table")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]