This is an automated email from the ASF dual-hosted git repository.

uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new d1828e710251 [SPARK-57815][SQL] Reject nanosecond-precision timestamps 
in Hive-serde tables
d1828e710251 is described below

commit d1828e710251d27d7245b5fedf48ccd313ecc585
Author: Stevo Mitric <[email protected]>
AuthorDate: Fri Jul 31 09:11:15 2026 +0000

    [SPARK-57815][SQL] Reject nanosecond-precision timestamps in Hive-serde 
tables
    
    ### What changes were proposed in this pull request?
    
    Handle nanosecond-precision timestamp columns (`TIMESTAMP_LTZ(p)` / 
`TIMESTAMP_NTZ(p)`, `p` in [7, 9]) in the Hive-serde (`STORED AS`) write path 
by rejecting them explicitly, mirroring how the TIME type is handled 
(SPARK-57556):
    
    - `HiveFileFormat.supportDataType` now returns `false` for 
`AnyTimestampNanoType` (recursing into nested types), so writing such a column 
fails with a clear `UNSUPPORTED_DATA_TYPE_FOR_DATASOURCE` error at 
`verifySchema` time instead of silently downgrading to microsecond precision.
    - `HiveInspectors.toInspector(dataType)`, `toInspector(expr)` (for 
`Literal`), and `toTypeInfo` now throw `UNSUPPORTED_DATATYPE` for 
`AnyTimestampNanoType`, since Hive has no nanosecond-precision timestamp type 
to map to.
    
    Rejection (rather than full support) is the intentional behavior chosen by 
the umbrella: Hive has no nanosecond timestamp type, and the 
metastore-compatibility side already treats these types as Hive-incompatible 
(SPARK-57831).
    
    ### Why are the changes needed?
    
    Sub-task of SPARK-56822. Without this, the Hive ORC serde silently 
downgrades nanosecond timestamps to microsecond `TimestampType` on write, 
losing precision with no warning. Silent downgrades must be eliminated.
    
    ### Does this PR introduce any user-facing change?
    
    Yes. Writing a nanosecond-precision timestamp column through the Hive serde 
now fails with a clear error instead of silently losing sub-microsecond 
precision.
    
    ### How was this patch tested?
    
    New unit test in `HiveInspectorSuite` covering `toInspector`/`toTypeInfo` 
for all four nanos type/precision combinations, and a new `InsertSuite` test 
that writes to a Hive-serde directory `STORED AS ORC` and `STORED AS PARQUET`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Code (Claude Opus 4.8)
    
    Closes #57635 from stevomitric/stevomitric/spark-57815-nanos.
    
    Authored-by: Stevo Mitric <[email protected]>
    Signed-off-by: Uros Bojanic <[email protected]>
---
 .../org/apache/spark/sql/hive/HiveInspectors.scala |  9 ++++++
 .../spark/sql/hive/execution/HiveFileFormat.scala  |  6 +++-
 .../apache/spark/sql/hive/HiveInspectorSuite.scala | 20 +++++++++++++
 .../org/apache/spark/sql/hive/InsertSuite.scala    | 34 ++++++++++++++++++++++
 4 files changed, 68 insertions(+), 1 deletion(-)

diff --git 
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala 
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 7f76132e2572..662e55dac543 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -967,6 +967,9 @@ private[hive] trait HiveInspectors {
       toInspector(sqlType)
     // Hive has no TIME type, so it cannot be represented by any Hive object 
inspector.
     case _: TimeType => throw unsupportedHiveType(dataType)
+    // Hive has no nanosecond-precision timestamp type, so it cannot be 
represented by any Hive
+    // object inspector. Reject it instead of silently downgrading to 
microsecond precision.
+    case _: AnyTimestampNanoType => throw unsupportedHiveType(dataType)
   }
 
   private def unsupportedHiveType(dataType: DataType): AnalysisException = {
@@ -1044,6 +1047,10 @@ private[hive] trait HiveInspectors {
     // Hive has no TIME type, so a TIME constant cannot be mapped to a Hive 
object inspector.
     case Literal(_, dt: TimeType) =>
       throw unsupportedHiveType(dt)
+    // Hive has no nanosecond-precision timestamp type, so such a constant 
cannot be mapped to a
+    // Hive object inspector.
+    case Literal(_, dt: AnyTimestampNanoType) =>
+      throw unsupportedHiveType(dt)
     // We will enumerate all of the possible constant expressions, throw 
exception if we missed
     case Literal(_, dt) =>
       throw SparkException.internalError(s"Hive doesn't support the constant 
type [$dt].")
@@ -1297,6 +1304,8 @@ private[hive] trait HiveInspectors {
       case _: YearMonthIntervalType => intervalYearMonthTypeInfo
       // Hive has no TIME type, so there is no Hive TypeInfo to map it to.
       case _: TimeType => throw unsupportedHiveType(dt)
+      // Hive has no nanosecond-precision timestamp type, so there is no Hive 
TypeInfo to map it to.
+      case _: AnyTimestampNanoType => throw unsupportedHiveType(dt)
       case dt => throw unsupportedHiveType(dt)
     }
   }
diff --git 
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
 
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
index acbc72fbf7e0..4545acc7f203 100644
--- 
a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
+++ 
b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveFileFormat.scala
@@ -41,7 +41,7 @@ import 
org.apache.spark.sql.execution.datasources.{FileFormat, OutputWriter, Out
 import org.apache.spark.sql.hive.{HiveInspectors, HiveTableUtil}
 import org.apache.spark.sql.internal.SessionStateHelper
 import org.apache.spark.sql.sources.DataSourceRegister
-import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType, 
TimeType, UserDefinedType}
+import org.apache.spark.sql.types.{AnyTimestampNanoType, ArrayType, DataType, 
MapType, StructType, TimeType, UserDefinedType}
 import org.apache.spark.util.SerializableJobConf
 
 /**
@@ -120,6 +120,10 @@ case class HiveFileFormat(fileSinkConf: FileSinkDesc)
     // (recursing into nested types) while preserving the default behavior for 
all other types.
     case _: TimeType => false
 
+    // Hive has no nanosecond-precision timestamp type. Reject it explicitly 
rather than silently
+    // downgrading to microsecond precision (which the Hive serde would 
otherwise do).
+    case _: AnyTimestampNanoType => false
+
     case st: StructType => st.forall { f => supportDataType(f.dataType) }
 
     case ArrayType(elementType, _) => supportDataType(elementType)
diff --git 
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala 
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index b7fb506f07b5..a88dcd0ef56a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -308,4 +308,24 @@ class HiveInspectorSuite extends SparkFunSuite with 
HiveInspectors {
       condition = "UNSUPPORTED_DATATYPE",
       parameters = expectedParams)
   }
+
+  test("SPARK-57815: nanosecond timestamp types are unsupported in Hive object 
inspectors") {
+    Seq(
+      TimestampNTZNanosType(9), TimestampNTZNanosType(7),
+      TimestampLTZNanosType(9), TimestampLTZNanosType(8)).foreach { nanosType 
=>
+      val expectedParams = Map("typeName" -> s"\"${nanosType.sql}\"")
+      checkError(
+        exception = intercept[AnalysisException](toInspector(nanosType)),
+        condition = "UNSUPPORTED_DATATYPE",
+        parameters = expectedParams)
+      checkError(
+        exception = 
intercept[AnalysisException](toInspector(Literal.create(null, nanosType))),
+        condition = "UNSUPPORTED_DATATYPE",
+        parameters = expectedParams)
+      checkError(
+        exception = intercept[AnalysisException](nanosType.toTypeInfo),
+        condition = "UNSUPPORTED_DATATYPE",
+        parameters = expectedParams)
+    }
+  }
 }
diff --git 
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala 
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala
index 7385410e1a71..db1743862e90 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala
@@ -708,6 +708,40 @@ class InsertSuite extends QueryTest with TestHiveSingleton 
with BeforeAndAfter {
     }
   }
 
+  test("SPARK-57815: nanosecond timestamp is unsupported when writing to a 
Hive serde directory") {
+    // Disable native data source conversion so that the write goes through 
the Hive serde path
+    // (HiveFileFormat) instead of a native data source that supports 
nanosecond timestamps.
+    withSQLConf(
+        HiveUtils.CONVERT_METASTORE_INSERT_DIR.key -> "false",
+        SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true") {
+      Seq("ORC", "PARQUET").foreach { fileFormat =>
+        Seq(
+          "TIMESTAMP_NTZ(9)" -> TimestampNTZNanosType(9),
+          "TIMESTAMP_LTZ(9)" -> TimestampLTZNanosType(9)).foreach { case 
(typeStr, dt) =>
+          withTempDir { dir =>
+            // InsertIntoHiveDirCommand wraps the failure in a SparkException, 
so assert on the
+            // cause. Rejecting here avoids a silent downgrade to microsecond 
precision.
+            val e = intercept[SparkException] {
+              sql(
+                s"""
+                   |INSERT OVERWRITE LOCAL DIRECTORY '${dir.toURI.getPath}'
+                   |STORED AS $fileFormat
+                   |SELECT CAST('2025-01-06 12:30:45.123456789' AS $typeStr) 
AS c
+                 """.stripMargin)
+            }
+            checkError(
+              exception = e.getCause.asInstanceOf[AnalysisException],
+              condition = "UNSUPPORTED_DATA_TYPE_FOR_DATASOURCE",
+              parameters = Map(
+                "columnName" -> "`c`",
+                "columnType" -> s"\"${dt.sql}\"",
+                "format" -> "Hive"))
+          }
+        }
+      }
+    }
+  }
+
   test("insert overwrite to dir from temp table") {
     withTempView("test_insert_table") {
       spark.range(10).selectExpr("id", "id AS 
str").createOrReplaceTempView("test_insert_table")


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to