This is an automated email from the ASF dual-hosted git repository.

uros-b pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 8be6c77b9c02 [SPARK-58466][SQL] Route ORC TIME/nanos support through 
the Types Framework
8be6c77b9c02 is described below

commit 8be6c77b9c02f53f44d26a263623eab580b2eeb3
Author: Stevo Mitric <[email protected]>
AuthorDate: Fri Jul 31 09:10:03 2026 +0000

    [SPARK-58466][SQL] Route ORC TIME/nanos support through the Types Framework
    
    ### What changes were proposed in this pull request?
    
    Introduce `OrcTypeOps` (mirroring `ParquetTypeOps`) so ORC 
read/write/pushdown for framework types is registered per-type instead of 
hardcoded across the ORC sources. Consolidates the inline `TimeType` / 
nanosecond-timestamp arms from `OrcUtils`, `OrcSerializer`, `OrcDeserializer`, 
and `OrcFilters` into three ops classes (`TimeTypeOrcOps`, 
`TimestampLTZNanosOrcOps`, `TimestampNTZNanosOrcOps`) and converts each call 
site to framework-first dispatch: `OrcTypeOps(dt).map(_.x).getOrElse(d [...]
    
    Scope is the row path only. Vectorized read (`OrcAtomicColumnVector`, a 
Java class with no per-type seam) and the nested stats min/max arm are left 
inline, matching `ParquetTypeOps` which also defers its vectorized-read hook. 
No `supportDataType` arm is needed (ORC already admits all `AtomicType`).
    
    ### Why are the changes needed?
    
    ORC was the last major storage format not routed through the Types 
Framework. A new framework type now gets ORC support by adding one ops class + 
one registry arm instead of editing four ORC sources.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Existing ORC suites, 198 tests: `OrcV1/V2QuerySuite`, 
`OrcV1/V2SourceSuite`, `OrcColumnarBatchReaderSuite`, `OrcFilterSuite` (incl. 
SPARK-57571 TIME pushdown and SPARK-57455 nanos round-trip). All pass unchanged.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Code (Claude Opus 4.8)
    
    Closes #56992 from stevomitric/stevomitric/orc-types-framework.
    
    Authored-by: Stevo Mitric <[email protected]>
    Signed-off-by: Uros Bojanic <[email protected]>
    (cherry picked from commit c2915c0564e617a976ef57be9a00afb57c91e4f6)
    Signed-off-by: Uros Bojanic <[email protected]>
---
 .../datasources/orc/OrcDeserializer.scala          |  18 +-
 .../sql/execution/datasources/orc/OrcFilters.scala |  18 +-
 .../execution/datasources/orc/OrcSerializer.scala  |  20 +--
 .../sql/execution/datasources/orc/OrcUtils.scala   |  25 ++-
 .../datasources/orc/types/ops/OrcTypeOps.scala     | 194 +++++++++++++++++++++
 .../datasources/orc/types/ops/TimeTypeOrcOps.scala |  87 +++++++++
 .../orc/types/ops/TimestampNanosOrcOps.scala       | 115 ++++++++++++
 7 files changed, 430 insertions(+), 47 deletions(-)

diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
index 04dd37dec50d..42fcb0cea2eb 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, 
UnsafeArrayData}
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns._
+import org.apache.spark.sql.execution.datasources.orc.types.ops.OrcTypeOps
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -126,7 +127,7 @@ class OrcDeserializer(
       case IntegerType | _: YearMonthIntervalType => (ordinal, value) =>
         updater.setInt(ordinal, value.asInstanceOf[IntWritable].get)
 
-      case LongType | _: DayTimeIntervalType | _: TimestampNTZType | _: 
TimeType =>
+      case LongType | _: DayTimeIntervalType | _: TimestampNTZType =>
         (ordinal, value) => updater.setLong(ordinal, 
value.asInstanceOf[LongWritable].get)
 
       case FloatType => (ordinal, value) =>
@@ -149,16 +150,11 @@ class OrcDeserializer(
 
       case TimestampType => (ordinal, value) =>
         updater.setLong(ordinal, 
DateTimeUtils.fromJavaTimestamp(value.asInstanceOf[OrcTimestamp]))
-      case t: TimestampLTZNanosType => (ordinal, value) =>
-        val ts = value.asInstanceOf[OrcTimestamp]
-        val instant = ts.toInstant
-        updater.set(ordinal, DateTimeUtils.instantToTimestampNanos(instant, 
t.precision))
-      case t: TimestampNTZNanosType => (ordinal, value) =>
-        val ts = value.asInstanceOf[OrcTimestamp]
-        val localDateTime = ts.toLocalDateTime
-        updater.set(
-          ordinal,
-          DateTimeUtils.localDateTimeToTimestampNanos(localDateTime, 
t.precision))
+
+      // Framework types (TimeType, nanosecond timestamps) provide their own 
ORC row reader. The
+      // setter callbacks decouple the ops sub-package from the sealed 
CatalystDataUpdater trait.
+      case OrcTypeOps(ops) =>
+        ops.makeDeserializer(updater.setLong, updater.set)
 
       case DecimalType.Fixed(precision, scale) => (ordinal, value) =>
         val v = OrcShimUtils.getDecimal(value)
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index 66770870dabe..4a092a792a79 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
-import java.time.{Duration, Instant, LocalDate, LocalDateTime, LocalTime, 
Period}
+import java.time.{Duration, Instant, LocalDate, LocalDateTime, Period}
 
 import org.apache.hadoop.hive.common.`type`.HiveDecimal
 import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
@@ -25,9 +25,10 @@ import 
org.apache.hadoop.hive.ql.io.sarg.SearchArgument.Builder
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
 import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable
 
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros, 
localDateTimeToMicros, localDateToDays, localTimeToNanos, toJavaDate, 
toJavaTimestamp}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros, 
localDateTimeToMicros, localDateToDays, toJavaDate, toJavaTimestamp}
 import org.apache.spark.sql.catalyst.util.IntervalUtils
 import org.apache.spark.sql.errors.QueryExecutionErrors
+import org.apache.spark.sql.execution.datasources.orc.types.ops.OrcTypeOps
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
@@ -142,13 +143,17 @@ private[sql] object OrcFilters extends OrcFiltersBase {
   def getPredicateLeafType(dataType: DataType): PredicateLeaf.Type = dataType 
match {
     case BooleanType => PredicateLeaf.Type.BOOLEAN
     case ByteType | ShortType | IntegerType | LongType |
-         _: AnsiIntervalType | TimestampNTZType | _: TimeType => 
PredicateLeaf.Type.LONG
+         _: AnsiIntervalType | TimestampNTZType => PredicateLeaf.Type.LONG
     case FloatType | DoubleType => PredicateLeaf.Type.FLOAT
     case StringType => PredicateLeaf.Type.STRING
     case DateType => PredicateLeaf.Type.DATE
     case TimestampType => PredicateLeaf.Type.TIMESTAMP
     case _: DecimalType => PredicateLeaf.Type.DECIMAL
-    case _ => throw 
QueryExecutionErrors.unsupportedOperationForDataTypeError(dataType)
+    // Framework types (e.g. TimeType) supply their own predicate-leaf type. A 
framework type whose
+    // predicateLeafType is None (like the nanos-timestamp types), or any 
other unmapped type,
+    // reaches the same unsupported-type error as before this change.
+    case dt => OrcTypeOps(dt).flatMap(_.predicateLeafType)
+      .getOrElse(throw 
QueryExecutionErrors.unsupportedOperationForDataTypeError(dt))
   }
 
   /**
@@ -170,12 +175,13 @@ private[sql] object OrcFilters extends OrcFiltersBase {
       toJavaTimestamp(instantToMicros(value.asInstanceOf[Instant]))
     case _: TimestampNTZType if value.isInstanceOf[LocalDateTime] =>
       localDateTimeToMicros(value.asInstanceOf[LocalDateTime])
-    case _: TimeType if value.isInstanceOf[LocalTime] =>
-      localTimeToNanos(value.asInstanceOf[LocalTime])
     case _: YearMonthIntervalType =>
       IntervalUtils.periodToMonths(value.asInstanceOf[Period]).longValue()
     case _: DayTimeIntervalType =>
       IntervalUtils.durationToMicros(value.asInstanceOf[Duration])
+    // Framework types (e.g. TimeType) cast their own filter literals; the 
default is identity, so
+    // non-framework types and non-matching values fall through unchanged.
+    case OrcTypeOps(ops) => ops.castFilterLiteral(value)
     case _ => value
   }
 
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcSerializer.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcSerializer.scala
index 64bc3e429212..358770064518 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcSerializer.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcSerializer.scala
@@ -27,8 +27,8 @@ import org.apache.spark.SparkException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.execution.datasources.orc.types.ops.OrcTypeOps
 import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.TimestampNanosVal
 
 /**
  * A serializer to serialize Spark rows to ORC structs.
@@ -107,7 +107,7 @@ class OrcSerializer(dataSchema: StructType) {
       }
 
 
-    case LongType | _: DayTimeIntervalType | _: TimestampNTZType | _: TimeType 
=>
+    case LongType | _: DayTimeIntervalType | _: TimestampNTZType =>
       if (reuseObj) {
         val result = new LongWritable()
         (getter, ordinal) =>
@@ -155,19 +155,9 @@ class OrcSerializer(dataSchema: StructType) {
       val result = new OrcTimestamp(ts.getTime)
       result.setNanos(ts.getNanos)
       result
-    case t: TimestampLTZNanosType => (getter, ordinal) =>
-      val v = getter.get(ordinal, t).asInstanceOf[TimestampNanosVal]
-      val instant = DateTimeUtils.timestampNanosToInstant(v)
-      val result = new OrcTimestamp(instant.toEpochMilli)
-      result.setNanos(instant.getNano)
-      result
-    case t: TimestampNTZNanosType => (getter, ordinal) =>
-      val v = getter.get(ordinal, t).asInstanceOf[TimestampNanosVal]
-      val localDateTime = DateTimeUtils.timestampNanosToLocalDateTime(v)
-      val ts = java.sql.Timestamp.valueOf(localDateTime)
-      val result = new OrcTimestamp(ts.getTime)
-      result.setNanos(ts.getNanos)
-      result
+
+    // Framework types (TimeType, nanosecond timestamps) provide their own ORC 
value writer.
+    case OrcTypeOps(ops) => ops.makeSerializer(reuseObj)
 
     case DecimalType.Fixed(precision, scale) =>
       OrcShimUtils.getHiveDecimalWritable(precision, scale)
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
index 1073b53dcaf6..e1676bd3ff51 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
@@ -48,6 +48,7 @@ import org.apache.spark.sql.catalyst.util.{quoteIdentifier, 
CaseInsensitiveMap,
 import org.apache.spark.sql.connector.expressions.aggregate.{Aggregation, 
Count, CountStar, Max, Min}
 import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.datasources.{AggregatePushDownUtils, 
SchemaMergeUtils}
+import org.apache.spark.sql.execution.datasources.orc.types.ops.OrcTypeOps
 import org.apache.spark.sql.execution.datasources.v2.V2ColumnUtils
 import org.apache.spark.sql.types._
 import org.apache.spark.util.ArrayImplicits._
@@ -350,11 +351,10 @@ object OrcUtils extends Logging {
       s"array<${getOrcSchemaString(a.elementType)}>"
     case m: MapType =>
       
s"map<${getOrcSchemaString(m.keyType)},${getOrcSchemaString(m.valueType)}>"
-    case _: DayTimeIntervalType | _: TimestampNTZType | _: TimeType => 
LongType.catalogString
-    case _: TimestampLTZNanosType => "timestamp with local time zone"
-    case _: TimestampNTZNanosType => "timestamp"
+    case _: DayTimeIntervalType | _: TimestampNTZType => LongType.catalogString
     case _: YearMonthIntervalType => IntegerType.catalogString
-    case _ => dt.catalogString
+    // Framework types (TimeType, nanosecond timestamps) supply their own ORC 
schema string.
+    case _ => OrcTypeOps(dt).map(_.orcSchemaString).getOrElse(dt.catalogString)
   }
 
   def orcTypeDescription(dt: DataType): TypeDescription = {
@@ -372,21 +372,16 @@ object OrcUtils extends Logging {
           val typeDesc = new TypeDescription(TypeDescription.Category.LONG)
           typeDesc.setAttribute(CATALYST_TYPE_ATTRIBUTE_NAME, n.typeName)
           Some(typeDesc)
-        case tm: TimeType =>
-          val typeDesc = new TypeDescription(TypeDescription.Category.LONG)
-          typeDesc.setAttribute(CATALYST_TYPE_ATTRIBUTE_NAME, tm.typeName)
-          Some(typeDesc)
         case t: TimestampType =>
           val typeDesc = new 
TypeDescription(TypeDescription.Category.TIMESTAMP)
           typeDesc.setAttribute(CATALYST_TYPE_ATTRIBUTE_NAME, t.typeName)
           Some(typeDesc)
-        case t: TimestampLTZNanosType =>
-          val typeDesc = new 
TypeDescription(TypeDescription.Category.TIMESTAMP_INSTANT)
-          typeDesc.setAttribute(CATALYST_TYPE_ATTRIBUTE_NAME, t.typeName)
-          Some(typeDesc)
-        case t: TimestampNTZNanosType =>
-          val typeDesc = new 
TypeDescription(TypeDescription.Category.TIMESTAMP)
-          typeDesc.setAttribute(CATALYST_TYPE_ATTRIBUTE_NAME, t.typeName)
+        // Framework types (TimeType, nanosecond timestamps) supply their own 
ORC category; the
+        // CATALYST_TYPE_ATTRIBUTE_NAME is stamped uniformly here so the true 
Spark type
+        // round-trips on read.
+        case OrcTypeOps(ops) =>
+          val typeDesc = new TypeDescription(ops.orcCategory)
+          typeDesc.setAttribute(CATALYST_TYPE_ATTRIBUTE_NAME, dt.typeName)
           Some(typeDesc)
         case _: StringType =>
           val typeDesc = new TypeDescription(TypeDescription.Category.STRING)
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/OrcTypeOps.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/OrcTypeOps.scala
new file mode 100644
index 000000000000..357d3b18bad8
--- /dev/null
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/OrcTypeOps.scala
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc.types.ops
+
+import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf
+import org.apache.hadoop.io.WritableComparable
+import org.apache.orc.TypeDescription
+
+import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
+import org.apache.spark.sql.types.{DataType, TimestampLTZNanosType, 
TimestampNTZNanosType, TimeType}
+
+/**
+ * Optional trait for ORC storage-format integration in the Types Framework.
+ *
+ * Implement this trait to enable ORC read/write support for a framework type. 
Each framework
+ * type that supports ORC provides a concrete implementation and registers it 
in the companion
+ * object's apply() method.
+ *
+ * The trait covers the ORC concerns that are homogeneous across the row-based 
code paths:
+ *   - Schema mapping: Spark DataType -> ORC schema string 
(OrcUtils.getOrcSchemaString) and the
+ *     ORC TypeDescription category (OrcUtils.orcTypeDescription). ORC has no 
native TIME /
+ *     nanosecond-timestamp category, so framework types map onto an existing 
physical ORC
+ *     category and round-trip the true Spark type via the 
CATALYST_TYPE_ATTRIBUTE_NAME attribute
+ *     (stamped uniformly by the caller, so the ops only chooses the category).
+ *   - Value write (serialize): Catalyst value -> ORC WritableComparable
+ *     (OrcSerializer.newConverter)
+ *   - Row-based read (deserialize): ORC WritableComparable -> Catalyst value
+ *     (OrcDeserializer.newWriter)
+ *   - Predicate pushdown: PredicateLeaf.Type mapping + literal casting 
(OrcFilters)
+ *
+ * DELIBERATELY NOT ON THE TRAIT:
+ *   - Vectorized read. ORC's vectorized path (OrcAtomicColumnVector, a Java 
class) dispatches via
+ *     boolean `instanceof` flags set in the constructor plus typed accessor 
methods
+ *     (getTimestampNTZNanos/getTimestampLTZNanos), NOT via an 
`Ops(dt).map(_.x)` closure. It has
+ *     no per-type extension seam, so it stays inline. This mirrors Parquet, 
whose vectorized read
+ *     is likewise not routed through ParquetTypeOps (it dispatches on the 
Spark type inline in
+ *     ParquetVectorUpdaterFactory.getUpdater).
+ *
+ *     CAVEAT for a new type author: the vectorized reader is a SEPARATE 
registration you must
+ *     handle in addition to the ops class. OrcUtils.supportColumnarReads 
returns true for every
+ *     `AtomicType` and OrcAtomicColumnVector dispatches by `instanceof`, so a 
new framework
+ *     `AtomicType` wired only into this ops registry is still routed to the 
vectorized reader,
+ *     where it would be read as raw physical values (silently wrong) rather 
than failing loudly.
+ *     Add an OrcAtomicColumnVector arm (or exclude the type from 
supportColumnarReads) as well.
+ *     The current types are wired correctly; this note is so the "one ops 
class + one registry
+ *     arm" framing does not mislead.
+ *   - supportDataType. OrcFileFormat.supportDataType / 
OrcTable.supportsDataType already admit
+ *     every `AtomicType` via `case _: AtomicType => true`, so framework types 
are supported with
+ *     no per-type arm; no gate method is needed (unlike Parquet, whose 
default differs).
+ *
+ * DISPATCH PATTERN: each ORC integration site keeps its existing 
built-in-type arms unchanged and
+ * routes only framework types through the ops. The built-in types are matched 
first; framework
+ * types are handled by an added arm reached after them (i.e. framework types 
are dispatched last,
+ * not first), so this change never alters how a built-in type is handled. Two 
shapes are used:
+ *   - Inside a `dataType match` (OrcSerializer.newConverter, 
OrcDeserializer.newWriter,
+ *     OrcUtils.orcTypeDescription, OrcFilters.castLiteralValue): an added
+ *     `case OrcTypeOps(ops) => ops.method(...)` arm placed among the existing 
arms. The `unapply`
+ *     extractor binds the ops in a single registry lookup.
+ *   - In expression position (OrcUtils.getOrcSchemaString, 
OrcFilters.getPredicateLeafType): the
+ *     original fallback stays inline and framework types are folded in with
+ *     `OrcTypeOps(dt).map(_.method).getOrElse(<original fallback>)`.
+ * There are no extracted `*Default` methods; the original ORC code is left in 
place as the
+ * fallback arm/expression.
+ *
+ * DECOUPLING NOTE: makeDeserializer takes the Catalyst setter callbacks it 
needs
+ * ((Int, Long) => Unit and (Int, Any) => Unit) rather than OrcDeserializer's 
CatalystDataUpdater,
+ * because that updater is a sealed trait nested in OrcDeserializer and is not 
visible to this
+ * sub-package. The callbacks are the only two setter shapes the current 
framework types use.
+ *
+ * @see TimeTypeOrcOps for a reference implementation (primitive Long-backed 
type)
+ * @see TimestampLTZNanosOrcOps / TimestampNTZNanosOrcOps for 
OrcTimestamp-backed types
+ * @since 5.0.0
+ */
+private[orc] trait OrcTypeOps extends Serializable {
+
+  // ==================== Schema Mapping ====================
+
+  /**
+   * The ORC schema-string fragment for this type 
(OrcUtils.getOrcSchemaString). Examples:
+   * TimeType -> "bigint" (LongType.catalogString), TimestampLTZNanosType -> 
"timestamp with local
+   * time zone", TimestampNTZNanosType -> "timestamp".
+   */
+  def orcSchemaString: String
+
+  /**
+   * The ORC TypeDescription category for this type 
(OrcUtils.orcTypeDescription). The caller
+   * stamps CATALYST_TYPE_ATTRIBUTE_NAME = sparkType.typeName onto the 
returned descriptor, so the
+   * ops only chooses the physical category.
+   */
+  def orcCategory: TypeDescription.Category
+
+  // ==================== Value Write (serialize) ====================
+
+  /**
+   * Creates a converter that turns a Catalyst value at an ordinal into an ORC 
WritableComparable
+   * (OrcSerializer.newConverter).
+   *
+   * @param reuseObj whether the serializer may reuse a single mutable 
Writable across rows
+   *                 (OrcSerializer passes this through; the primitive 
Long-backed TimeType reuses,
+   *                 the OrcTimestamp-backed nanos types do not).
+   */
+  def makeSerializer(reuseObj: Boolean): (SpecializedGetters, Int) => 
WritableComparable[_]
+
+  // ==================== Row-Based Read (deserialize) ====================
+
+  /**
+   * Creates a writer that sets a decoded ORC value into the Catalyst row at 
an ordinal
+   * (OrcDeserializer.newWriter). The WritableComparable is the raw ORC value 
(LongWritable for
+   * LONG-category types, OrcTimestamp for the timestamp categories).
+   *
+   * @param setLong callback into the row's setLong (used by primitive 
Long-backed types)
+   * @param set     callback into the row's generic set (used by object-backed 
types, e.g. nanos)
+   */
+  def makeDeserializer(
+      setLong: (Int, Long) => Unit,
+      set: (Int, Any) => Unit): (Int, WritableComparable[_]) => Unit
+
+  // ==================== Predicate Pushdown ====================
+
+  /**
+   * The ORC PredicateLeaf.Type for this type, consumed by 
OrcFilters.getPredicateLeafType. Return
+   * Some(type) to enable predicate pushdown for this type.
+   *
+   * A None keeps the pre-existing OrcFilters behavior for a type with no leaf 
mapping: it reaches
+   * getPredicateLeafType's final arm, which throws 
unsupportedOperationForDataTypeError. In
+   * practice that arm is only reached for a column already deemed 
pushdown-eligible upstream
+   * (OrcFiltersBase.getSearchableTypeMap admits any AtomicType), so a 
framework AtomicType that
+   * returns None and is used in a pushed filter would fail during planning. A 
future type that
+   * must be pushdown-eligible therefore has to return Some here (and, if its 
literal needs
+   * conversion, override castFilterLiteral); returning None is only safe for 
a type that can never
+   * reach getPredicateLeafType.
+   *
+   * TimeType returns Some(LONG). The nanosecond-timestamp types return None: 
they had no
+   * getPredicateLeafType arm before this change either, so the behavior 
(throw if ever reached) is
+   * preserved, not newly introduced.
+   */
+  def predicateLeafType: Option[PredicateLeaf.Type] = None
+
+  /**
+   * Casts a filter literal to the ORC search-argument representation 
(OrcFilters.castLiteralValue).
+   * Only meaningful for types that also return a predicateLeafType. Default 
is identity.
+   */
+  def castFilterLiteral(value: Any): Any = value
+}
+
+/**
+ * Factory object for creating OrcTypeOps instances.
+ *
+ * Provides forward lookup (DataType -> ops) for the framework-type dispatch 
arms at ORC
+ * integration sites. apply() returns Some only for framework-managed types, 
so callers fall back
+ * to the original inline path for everything else.
+ */
+private[orc] object OrcTypeOps {
+
+  /**
+   * Returns an OrcTypeOps instance for the given DataType, if supported.
+   *
+   * Returns None if the type has no ORC ops. This is the single registration 
point for all ORC
+   * type operations.
+   */
+  def apply(dt: DataType): Option[OrcTypeOps] = dt match {
+    case tt: TimeType => Some(TimeTypeOrcOps(tt))
+    case t: TimestampLTZNanosType => Some(TimestampLTZNanosOrcOps(t))
+    case t: TimestampNTZNanosType => Some(TimestampNTZNanosOrcOps(t))
+    // Add new types here - single registration point
+    case _ => None
+  }
+
+  /**
+   * Extractor so a `dataType match` arm can bind the ops in a single lookup:
+   * {{{
+   *   case OrcTypeOps(ops) => ops.method(...)
+   * }}}
+   * Delegates to apply(); returning the same Option keeps the registry the 
single source of truth
+   * and avoids the double lookup of a `case _ if OrcTypeOps(dt).isDefined => 
OrcTypeOps(dt).get`
+   * guard.
+   */
+  def unapply(dt: DataType): Option[OrcTypeOps] = apply(dt)
+}
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/TimeTypeOrcOps.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/TimeTypeOrcOps.scala
new file mode 100644
index 000000000000..1896a2712eec
--- /dev/null
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/TimeTypeOrcOps.scala
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc.types.ops
+
+import java.time.LocalTime
+
+import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf
+import org.apache.hadoop.io.{LongWritable, WritableComparable}
+import org.apache.orc.TypeDescription
+
+import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.localTimeToNanos
+import org.apache.spark.sql.types.{LongType, TimeType}
+
+/**
+ * ORC operations for TimeType.
+ *
+ * TimeType is a primitive Long-backed type (nanoseconds since midnight). ORC 
has no TIME category,
+ * so it is stored in the LONG physical category with the true Spark type 
recovered on read from
+ * the CATALYST_TYPE_ATTRIBUTE_NAME attribute. Read and write are pure Long 
pass-through: the
+ * internal nanos-of-day value IS the stored LONG (no unit conversion, unlike 
Parquet which may
+ * store MICROS). Precision affects only display, so nothing is truncated here.
+ *
+ * @param t the TimeType (precision is unused on the ORC path; kept for 
symmetry with the other
+ *          storage ops and possible future precision-aware behavior).
+ * @since 5.0.0
+ */
+case class TimeTypeOrcOps(t: TimeType) extends OrcTypeOps {
+
+  // ==================== Schema Mapping ====================
+
+  // Was: OrcUtils.getOrcSchemaString  `case ... | _: TimeType => 
LongType.catalogString`
+  override def orcSchemaString: String = LongType.catalogString
+
+  // Was: OrcUtils.orcTypeDescription  `case tm: TimeType => new 
TypeDescription(LONG) ...`
+  override def orcCategory: TypeDescription.Category = 
TypeDescription.Category.LONG
+
+  // ==================== Value Write (serialize) ====================
+
+  // Was: OrcSerializer.newConverter  `case ... | _: TimeType => ... 
LongWritable ...`
+  override def makeSerializer(
+      reuseObj: Boolean): (SpecializedGetters, Int) => WritableComparable[_] =
+    if (reuseObj) {
+      val result = new LongWritable()
+      (getter: SpecializedGetters, ordinal: Int) => {
+        result.set(getter.getLong(ordinal))
+        result
+      }
+    } else {
+      (getter: SpecializedGetters, ordinal: Int) => new 
LongWritable(getter.getLong(ordinal))
+    }
+
+  // ==================== Row-Based Read (deserialize) ====================
+
+  // Was: OrcDeserializer.newWriter  `case ... | _: TimeType => ... setLong 
...`
+  override def makeDeserializer(
+      setLong: (Int, Long) => Unit,
+      set: (Int, Any) => Unit): (Int, WritableComparable[_]) => Unit =
+    (ordinal: Int, value: WritableComparable[_]) =>
+      setLong(ordinal, value.asInstanceOf[LongWritable].get)
+
+  // ==================== Predicate Pushdown ====================
+
+  // Was: OrcFilters.getPredicateLeafType  `case ... | _: TimeType => 
PredicateLeaf.Type.LONG`
+  override def predicateLeafType: Option[PredicateLeaf.Type] = 
Some(PredicateLeaf.Type.LONG)
+
+  // Was: OrcFilters.castLiteralValue  `case _: TimeType if 
value.isInstanceOf[LocalTime] => ...`
+  override def castFilterLiteral(value: Any): Any = value match {
+    case lt: LocalTime => localTimeToNanos(lt)
+    case other => other
+  }
+}
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/TimestampNanosOrcOps.scala
 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/TimestampNanosOrcOps.scala
new file mode 100644
index 000000000000..87a4bd252a88
--- /dev/null
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/TimestampNanosOrcOps.scala
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc.types.ops
+
+import org.apache.hadoop.io.WritableComparable
+import org.apache.orc.TypeDescription
+import org.apache.orc.mapred.OrcTimestamp
+
+import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.types.{TimestampLTZNanosType, 
TimestampNTZNanosType}
+import org.apache.spark.unsafe.types.TimestampNanosVal
+
+/**
+ * ORC operations for TimestampLTZNanosType (nanosecond precision, with time 
zone).
+ *
+ * Stored in ORC as the TIMESTAMP_INSTANT category (matching TIMESTAMP with 
local time zone). The
+ * internal TimestampNanosVal is converted through java.time.Instant in both 
directions; the true
+ * Spark type and precision are recovered on read from the 
CATALYST_TYPE_ATTRIBUTE_NAME attribute.
+ * ORC does not reuse the OrcTimestamp object (the conversion is already 
expensive), so the
+ * serializer runs with reuse disabled regardless of the reuseObj hint.
+ *
+ * No predicate pushdown: the inline OrcFilters code never added a nanos arm, 
so predicateLeafType
+ * stays None (inherited default).
+ *
+ * @since 5.0.0
+ */
+case class TimestampLTZNanosOrcOps(t: TimestampLTZNanosType) extends 
OrcTypeOps {
+
+  // Was: OrcUtils.getOrcSchemaString
+  // `case _: TimestampLTZNanosType => "timestamp with local time zone"`
+  override def orcSchemaString: String = "timestamp with local time zone"
+
+  // Was: OrcUtils.orcTypeDescription  `case t: TimestampLTZNanosType => ... 
TIMESTAMP_INSTANT ...`
+  override def orcCategory: TypeDescription.Category = 
TypeDescription.Category.TIMESTAMP_INSTANT
+
+  // Was: OrcSerializer.newConverter  `case t: TimestampLTZNanosType => ...`
+  override def makeSerializer(
+      reuseObj: Boolean): (SpecializedGetters, Int) => WritableComparable[_] =
+    (getter: SpecializedGetters, ordinal: Int) => {
+      val v = getter.get(ordinal, t).asInstanceOf[TimestampNanosVal]
+      val instant = DateTimeUtils.timestampNanosToInstant(v)
+      val result = new OrcTimestamp(instant.toEpochMilli)
+      result.setNanos(instant.getNano)
+      result
+    }
+
+  // Was: OrcDeserializer.newWriter  `case t: TimestampLTZNanosType => ...`
+  override def makeDeserializer(
+      setLong: (Int, Long) => Unit,
+      set: (Int, Any) => Unit): (Int, WritableComparable[_]) => Unit =
+    (ordinal: Int, value: WritableComparable[_]) => {
+      val ts = value.asInstanceOf[OrcTimestamp]
+      val instant = ts.toInstant
+      set(ordinal, DateTimeUtils.instantToTimestampNanos(instant, t.precision))
+    }
+}
+
+/**
+ * ORC operations for TimestampNTZNanosType (nanosecond precision, without 
time zone).
+ *
+ * Stored in ORC as the TIMESTAMP category. The internal TimestampNanosVal is 
converted through
+ * java.time.LocalDateTime in both directions; the true Spark type and 
precision are recovered on
+ * read from the CATALYST_TYPE_ATTRIBUTE_NAME attribute. As with the LTZ 
variant, the OrcTimestamp
+ * object is not reused.
+ *
+ * No predicate pushdown (predicateLeafType stays None).
+ *
+ * @since 5.0.0
+ */
+case class TimestampNTZNanosOrcOps(t: TimestampNTZNanosType) extends 
OrcTypeOps {
+
+  // Was: OrcUtils.getOrcSchemaString  `case _: TimestampNTZNanosType => 
"timestamp"`
+  override def orcSchemaString: String = "timestamp"
+
+  // Was: OrcUtils.orcTypeDescription  `case t: TimestampNTZNanosType => ... 
TIMESTAMP ...`
+  override def orcCategory: TypeDescription.Category = 
TypeDescription.Category.TIMESTAMP
+
+  // Was: OrcSerializer.newConverter  `case t: TimestampNTZNanosType => ...`
+  override def makeSerializer(
+      reuseObj: Boolean): (SpecializedGetters, Int) => WritableComparable[_] =
+    (getter: SpecializedGetters, ordinal: Int) => {
+      val v = getter.get(ordinal, t).asInstanceOf[TimestampNanosVal]
+      val localDateTime = DateTimeUtils.timestampNanosToLocalDateTime(v)
+      val ts = java.sql.Timestamp.valueOf(localDateTime)
+      val result = new OrcTimestamp(ts.getTime)
+      result.setNanos(ts.getNanos)
+      result
+    }
+
+  // Was: OrcDeserializer.newWriter  `case t: TimestampNTZNanosType => ...`
+  override def makeDeserializer(
+      setLong: (Int, Long) => Unit,
+      set: (Int, Any) => Unit): (Int, WritableComparable[_]) => Unit =
+    (ordinal: Int, value: WritableComparable[_]) => {
+      val ts = value.asInstanceOf[OrcTimestamp]
+      val localDateTime = ts.toLocalDateTime
+      set(ordinal, DateTimeUtils.localDateTimeToTimestampNanos(localDateTime, 
t.precision))
+    }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to