This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 8be6c77b9c02 [SPARK-58466][SQL] Route ORC TIME/nanos support through
the Types Framework
8be6c77b9c02 is described below
commit 8be6c77b9c02f53f44d26a263623eab580b2eeb3
Author: Stevo Mitric <[email protected]>
AuthorDate: Fri Jul 31 09:10:03 2026 +0000
[SPARK-58466][SQL] Route ORC TIME/nanos support through the Types Framework
### What changes were proposed in this pull request?
Introduce `OrcTypeOps` (mirroring `ParquetTypeOps`) so ORC
read/write/pushdown for framework types is registered per-type instead of
hardcoded across the ORC sources. Consolidates the inline `TimeType` /
nanosecond-timestamp arms from `OrcUtils`, `OrcSerializer`, `OrcDeserializer`,
and `OrcFilters` into three ops classes (`TimeTypeOrcOps`,
`TimestampLTZNanosOrcOps`, `TimestampNTZNanosOrcOps`) and converts each call
site to framework-first dispatch: `OrcTypeOps(dt).map(_.x).getOrElse(d [...]
Scope is the row path only. Vectorized read (`OrcAtomicColumnVector`, a
Java class with no per-type seam) and the nested stats min/max arm are left
inline, matching `ParquetTypeOps` which also defers its vectorized-read hook.
No `supportDataType` arm is needed (ORC already admits all `AtomicType`).
### Why are the changes needed?
ORC was the last major storage format not routed through the Types
Framework. A new framework type now gets ORC support by adding one ops class +
one registry arm instead of editing four ORC sources.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing ORC suites, 198 tests: `OrcV1/V2QuerySuite`,
`OrcV1/V2SourceSuite`, `OrcColumnarBatchReaderSuite`, `OrcFilterSuite` (incl.
SPARK-57571 TIME pushdown and SPARK-57455 nanos round-trip). All pass unchanged.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 4.8)
Closes #56992 from stevomitric/stevomitric/orc-types-framework.
Authored-by: Stevo Mitric <[email protected]>
Signed-off-by: Uros Bojanic <[email protected]>
(cherry picked from commit c2915c0564e617a976ef57be9a00afb57c91e4f6)
Signed-off-by: Uros Bojanic <[email protected]>
---
.../datasources/orc/OrcDeserializer.scala | 18 +-
.../sql/execution/datasources/orc/OrcFilters.scala | 18 +-
.../execution/datasources/orc/OrcSerializer.scala | 20 +--
.../sql/execution/datasources/orc/OrcUtils.scala | 25 ++-
.../datasources/orc/types/ops/OrcTypeOps.scala | 194 +++++++++++++++++++++
.../datasources/orc/types/ops/TimeTypeOrcOps.scala | 87 +++++++++
.../orc/types/ops/TimestampNanosOrcOps.scala | 115 ++++++++++++
7 files changed, 430 insertions(+), 47 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
index 04dd37dec50d..42fcb0cea2eb 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow,
UnsafeArrayData}
import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns._
+import org.apache.spark.sql.execution.datasources.orc.types.ops.OrcTypeOps
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
@@ -126,7 +127,7 @@ class OrcDeserializer(
case IntegerType | _: YearMonthIntervalType => (ordinal, value) =>
updater.setInt(ordinal, value.asInstanceOf[IntWritable].get)
- case LongType | _: DayTimeIntervalType | _: TimestampNTZType | _:
TimeType =>
+ case LongType | _: DayTimeIntervalType | _: TimestampNTZType =>
(ordinal, value) => updater.setLong(ordinal,
value.asInstanceOf[LongWritable].get)
case FloatType => (ordinal, value) =>
@@ -149,16 +150,11 @@ class OrcDeserializer(
case TimestampType => (ordinal, value) =>
updater.setLong(ordinal,
DateTimeUtils.fromJavaTimestamp(value.asInstanceOf[OrcTimestamp]))
- case t: TimestampLTZNanosType => (ordinal, value) =>
- val ts = value.asInstanceOf[OrcTimestamp]
- val instant = ts.toInstant
- updater.set(ordinal, DateTimeUtils.instantToTimestampNanos(instant,
t.precision))
- case t: TimestampNTZNanosType => (ordinal, value) =>
- val ts = value.asInstanceOf[OrcTimestamp]
- val localDateTime = ts.toLocalDateTime
- updater.set(
- ordinal,
- DateTimeUtils.localDateTimeToTimestampNanos(localDateTime,
t.precision))
+
+ // Framework types (TimeType, nanosecond timestamps) provide their own
ORC row reader. The
+ // setter callbacks decouple the ops sub-package from the sealed
CatalystDataUpdater trait.
+ case OrcTypeOps(ops) =>
+ ops.makeDeserializer(updater.setLong, updater.set)
case DecimalType.Fixed(precision, scale) => (ordinal, value) =>
val v = OrcShimUtils.getDecimal(value)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index 66770870dabe..4a092a792a79 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.execution.datasources.orc
-import java.time.{Duration, Instant, LocalDate, LocalDateTime, LocalTime,
Period}
+import java.time.{Duration, Instant, LocalDate, LocalDateTime, Period}
import org.apache.hadoop.hive.common.`type`.HiveDecimal
import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
@@ -25,9 +25,10 @@ import
org.apache.hadoop.hive.ql.io.sarg.SearchArgument.Builder
import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros,
localDateTimeToMicros, localDateToDays, localTimeToNanos, toJavaDate,
toJavaTimestamp}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros,
localDateTimeToMicros, localDateToDays, toJavaDate, toJavaTimestamp}
import org.apache.spark.sql.catalyst.util.IntervalUtils
import org.apache.spark.sql.errors.QueryExecutionErrors
+import org.apache.spark.sql.execution.datasources.orc.types.ops.OrcTypeOps
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types._
@@ -142,13 +143,17 @@ private[sql] object OrcFilters extends OrcFiltersBase {
def getPredicateLeafType(dataType: DataType): PredicateLeaf.Type = dataType
match {
case BooleanType => PredicateLeaf.Type.BOOLEAN
case ByteType | ShortType | IntegerType | LongType |
- _: AnsiIntervalType | TimestampNTZType | _: TimeType =>
PredicateLeaf.Type.LONG
+ _: AnsiIntervalType | TimestampNTZType => PredicateLeaf.Type.LONG
case FloatType | DoubleType => PredicateLeaf.Type.FLOAT
case StringType => PredicateLeaf.Type.STRING
case DateType => PredicateLeaf.Type.DATE
case TimestampType => PredicateLeaf.Type.TIMESTAMP
case _: DecimalType => PredicateLeaf.Type.DECIMAL
- case _ => throw
QueryExecutionErrors.unsupportedOperationForDataTypeError(dataType)
+ // Framework types (e.g. TimeType) supply their own predicate-leaf type. A
framework type whose
+ // predicateLeafType is None (like the nanos-timestamp types), or any
other unmapped type,
+ // reaches the same unsupported-type error as before this change.
+ case dt => OrcTypeOps(dt).flatMap(_.predicateLeafType)
+ .getOrElse(throw
QueryExecutionErrors.unsupportedOperationForDataTypeError(dt))
}
/**
@@ -170,12 +175,13 @@ private[sql] object OrcFilters extends OrcFiltersBase {
toJavaTimestamp(instantToMicros(value.asInstanceOf[Instant]))
case _: TimestampNTZType if value.isInstanceOf[LocalDateTime] =>
localDateTimeToMicros(value.asInstanceOf[LocalDateTime])
- case _: TimeType if value.isInstanceOf[LocalTime] =>
- localTimeToNanos(value.asInstanceOf[LocalTime])
case _: YearMonthIntervalType =>
IntervalUtils.periodToMonths(value.asInstanceOf[Period]).longValue()
case _: DayTimeIntervalType =>
IntervalUtils.durationToMicros(value.asInstanceOf[Duration])
+ // Framework types (e.g. TimeType) cast their own filter literals; the
default is identity, so
+ // non-framework types and non-matching values fall through unchanged.
+ case OrcTypeOps(ops) => ops.castFilterLiteral(value)
case _ => value
}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcSerializer.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcSerializer.scala
index 64bc3e429212..358770064518 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcSerializer.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcSerializer.scala
@@ -27,8 +27,8 @@ import org.apache.spark.SparkException
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.execution.datasources.orc.types.ops.OrcTypeOps
import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.TimestampNanosVal
/**
* A serializer to serialize Spark rows to ORC structs.
@@ -107,7 +107,7 @@ class OrcSerializer(dataSchema: StructType) {
}
- case LongType | _: DayTimeIntervalType | _: TimestampNTZType | _: TimeType
=>
+ case LongType | _: DayTimeIntervalType | _: TimestampNTZType =>
if (reuseObj) {
val result = new LongWritable()
(getter, ordinal) =>
@@ -155,19 +155,9 @@ class OrcSerializer(dataSchema: StructType) {
val result = new OrcTimestamp(ts.getTime)
result.setNanos(ts.getNanos)
result
- case t: TimestampLTZNanosType => (getter, ordinal) =>
- val v = getter.get(ordinal, t).asInstanceOf[TimestampNanosVal]
- val instant = DateTimeUtils.timestampNanosToInstant(v)
- val result = new OrcTimestamp(instant.toEpochMilli)
- result.setNanos(instant.getNano)
- result
- case t: TimestampNTZNanosType => (getter, ordinal) =>
- val v = getter.get(ordinal, t).asInstanceOf[TimestampNanosVal]
- val localDateTime = DateTimeUtils.timestampNanosToLocalDateTime(v)
- val ts = java.sql.Timestamp.valueOf(localDateTime)
- val result = new OrcTimestamp(ts.getTime)
- result.setNanos(ts.getNanos)
- result
+
+ // Framework types (TimeType, nanosecond timestamps) provide their own ORC
value writer.
+ case OrcTypeOps(ops) => ops.makeSerializer(reuseObj)
case DecimalType.Fixed(precision, scale) =>
OrcShimUtils.getHiveDecimalWritable(precision, scale)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
index 1073b53dcaf6..e1676bd3ff51 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
@@ -48,6 +48,7 @@ import org.apache.spark.sql.catalyst.util.{quoteIdentifier,
CaseInsensitiveMap,
import org.apache.spark.sql.connector.expressions.aggregate.{Aggregation,
Count, CountStar, Max, Min}
import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.execution.datasources.{AggregatePushDownUtils,
SchemaMergeUtils}
+import org.apache.spark.sql.execution.datasources.orc.types.ops.OrcTypeOps
import org.apache.spark.sql.execution.datasources.v2.V2ColumnUtils
import org.apache.spark.sql.types._
import org.apache.spark.util.ArrayImplicits._
@@ -350,11 +351,10 @@ object OrcUtils extends Logging {
s"array<${getOrcSchemaString(a.elementType)}>"
case m: MapType =>
s"map<${getOrcSchemaString(m.keyType)},${getOrcSchemaString(m.valueType)}>"
- case _: DayTimeIntervalType | _: TimestampNTZType | _: TimeType =>
LongType.catalogString
- case _: TimestampLTZNanosType => "timestamp with local time zone"
- case _: TimestampNTZNanosType => "timestamp"
+ case _: DayTimeIntervalType | _: TimestampNTZType => LongType.catalogString
case _: YearMonthIntervalType => IntegerType.catalogString
- case _ => dt.catalogString
+ // Framework types (TimeType, nanosecond timestamps) supply their own ORC
schema string.
+ case _ => OrcTypeOps(dt).map(_.orcSchemaString).getOrElse(dt.catalogString)
}
def orcTypeDescription(dt: DataType): TypeDescription = {
@@ -372,21 +372,16 @@ object OrcUtils extends Logging {
val typeDesc = new TypeDescription(TypeDescription.Category.LONG)
typeDesc.setAttribute(CATALYST_TYPE_ATTRIBUTE_NAME, n.typeName)
Some(typeDesc)
- case tm: TimeType =>
- val typeDesc = new TypeDescription(TypeDescription.Category.LONG)
- typeDesc.setAttribute(CATALYST_TYPE_ATTRIBUTE_NAME, tm.typeName)
- Some(typeDesc)
case t: TimestampType =>
val typeDesc = new
TypeDescription(TypeDescription.Category.TIMESTAMP)
typeDesc.setAttribute(CATALYST_TYPE_ATTRIBUTE_NAME, t.typeName)
Some(typeDesc)
- case t: TimestampLTZNanosType =>
- val typeDesc = new
TypeDescription(TypeDescription.Category.TIMESTAMP_INSTANT)
- typeDesc.setAttribute(CATALYST_TYPE_ATTRIBUTE_NAME, t.typeName)
- Some(typeDesc)
- case t: TimestampNTZNanosType =>
- val typeDesc = new
TypeDescription(TypeDescription.Category.TIMESTAMP)
- typeDesc.setAttribute(CATALYST_TYPE_ATTRIBUTE_NAME, t.typeName)
+ // Framework types (TimeType, nanosecond timestamps) supply their own
ORC category; the
+ // CATALYST_TYPE_ATTRIBUTE_NAME is stamped uniformly here so the true
Spark type
+ // round-trips on read.
+ case OrcTypeOps(ops) =>
+ val typeDesc = new TypeDescription(ops.orcCategory)
+ typeDesc.setAttribute(CATALYST_TYPE_ATTRIBUTE_NAME, dt.typeName)
Some(typeDesc)
case _: StringType =>
val typeDesc = new TypeDescription(TypeDescription.Category.STRING)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/OrcTypeOps.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/OrcTypeOps.scala
new file mode 100644
index 000000000000..357d3b18bad8
--- /dev/null
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/OrcTypeOps.scala
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc.types.ops
+
+import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf
+import org.apache.hadoop.io.WritableComparable
+import org.apache.orc.TypeDescription
+
+import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
+import org.apache.spark.sql.types.{DataType, TimestampLTZNanosType,
TimestampNTZNanosType, TimeType}
+
+/**
+ * Optional trait for ORC storage-format integration in the Types Framework.
+ *
+ * Implement this trait to enable ORC read/write support for a framework type.
Each framework
+ * type that supports ORC provides a concrete implementation and registers it
in the companion
+ * object's apply() method.
+ *
+ * The trait covers the ORC concerns that are homogeneous across the row-based
code paths:
+ * - Schema mapping: Spark DataType -> ORC schema string
(OrcUtils.getOrcSchemaString) and the
+ * ORC TypeDescription category (OrcUtils.orcTypeDescription). ORC has no
native TIME /
+ * nanosecond-timestamp category, so framework types map onto an existing
physical ORC
+ * category and round-trip the true Spark type via the
CATALYST_TYPE_ATTRIBUTE_NAME attribute
+ * (stamped uniformly by the caller, so the ops only chooses the category).
+ * - Value write (serialize): Catalyst value -> ORC WritableComparable
+ * (OrcSerializer.newConverter)
+ * - Row-based read (deserialize): ORC WritableComparable -> Catalyst value
+ * (OrcDeserializer.newWriter)
+ * - Predicate pushdown: PredicateLeaf.Type mapping + literal casting
(OrcFilters)
+ *
+ * DELIBERATELY NOT ON THE TRAIT:
+ * - Vectorized read. ORC's vectorized path (OrcAtomicColumnVector, a Java
class) dispatches via
+ * boolean `instanceof` flags set in the constructor plus typed accessor
methods
+ * (getTimestampNTZNanos/getTimestampLTZNanos), NOT via an
`Ops(dt).map(_.x)` closure. It has
+ * no per-type extension seam, so it stays inline. This mirrors Parquet,
whose vectorized read
+ * is likewise not routed through ParquetTypeOps (it dispatches on the
Spark type inline in
+ * ParquetVectorUpdaterFactory.getUpdater).
+ *
+ * CAVEAT for a new type author: the vectorized reader is a SEPARATE
registration you must
+ * handle in addition to the ops class. OrcUtils.supportColumnarReads
returns true for every
+ * `AtomicType` and OrcAtomicColumnVector dispatches by `instanceof`, so a
new framework
+ * `AtomicType` wired only into this ops registry is still routed to the
vectorized reader,
+ * where it would be read as raw physical values (silently wrong) rather
than failing loudly.
+ * Add an OrcAtomicColumnVector arm (or exclude the type from
supportColumnarReads) as well.
+ * The current types are wired correctly; this note is so the "one ops
class + one registry
+ * arm" framing does not mislead.
+ * - supportDataType. OrcFileFormat.supportDataType /
OrcTable.supportsDataType already admit
+ * every `AtomicType` via `case _: AtomicType => true`, so framework types
are supported with
+ * no per-type arm; no gate method is needed (unlike Parquet, whose
default differs).
+ *
+ * DISPATCH PATTERN: each ORC integration site keeps its existing
built-in-type arms unchanged and
+ * routes only framework types through the ops. The built-in types are matched
first; framework
+ * types are handled by an added arm reached after them (i.e. framework types
are dispatched last,
+ * not first), so this change never alters how a built-in type is handled. Two
shapes are used:
+ * - Inside a `dataType match` (OrcSerializer.newConverter,
OrcDeserializer.newWriter,
+ * OrcUtils.orcTypeDescription, OrcFilters.castLiteralValue): an added
+ * `case OrcTypeOps(ops) => ops.method(...)` arm placed among the existing
arms. The `unapply`
+ * extractor binds the ops in a single registry lookup.
+ * - In expression position (OrcUtils.getOrcSchemaString,
OrcFilters.getPredicateLeafType): the
+ * original fallback stays inline and framework types are folded in with
+ * `OrcTypeOps(dt).map(_.method).getOrElse(<original fallback>)`.
+ * There are no extracted `*Default` methods; the original ORC code is left in
place as the
+ * fallback arm/expression.
+ *
+ * DECOUPLING NOTE: makeDeserializer takes the Catalyst setter callbacks it
needs
+ * ((Int, Long) => Unit and (Int, Any) => Unit) rather than OrcDeserializer's
CatalystDataUpdater,
+ * because that updater is a sealed trait nested in OrcDeserializer and is not
visible to this
+ * sub-package. The callbacks are the only two setter shapes the current
framework types use.
+ *
+ * @see TimeTypeOrcOps for a reference implementation (primitive Long-backed
type)
+ * @see TimestampLTZNanosOrcOps / TimestampNTZNanosOrcOps for
OrcTimestamp-backed types
+ * @since 5.0.0
+ */
+private[orc] trait OrcTypeOps extends Serializable {
+
+ // ==================== Schema Mapping ====================
+
+ /**
+ * The ORC schema-string fragment for this type
(OrcUtils.getOrcSchemaString). Examples:
+ * TimeType -> "bigint" (LongType.catalogString), TimestampLTZNanosType ->
"timestamp with local
+ * time zone", TimestampNTZNanosType -> "timestamp".
+ */
+ def orcSchemaString: String
+
+ /**
+ * The ORC TypeDescription category for this type
(OrcUtils.orcTypeDescription). The caller
+ * stamps CATALYST_TYPE_ATTRIBUTE_NAME = sparkType.typeName onto the
returned descriptor, so the
+ * ops only chooses the physical category.
+ */
+ def orcCategory: TypeDescription.Category
+
+ // ==================== Value Write (serialize) ====================
+
+ /**
+ * Creates a converter that turns a Catalyst value at an ordinal into an ORC
WritableComparable
+ * (OrcSerializer.newConverter).
+ *
+ * @param reuseObj whether the serializer may reuse a single mutable
Writable across rows
+ * (OrcSerializer passes this through; the primitive
Long-backed TimeType reuses,
+ * the OrcTimestamp-backed nanos types do not).
+ */
+ def makeSerializer(reuseObj: Boolean): (SpecializedGetters, Int) =>
WritableComparable[_]
+
+ // ==================== Row-Based Read (deserialize) ====================
+
+ /**
+ * Creates a writer that sets a decoded ORC value into the Catalyst row at
an ordinal
+ * (OrcDeserializer.newWriter). The WritableComparable is the raw ORC value
(LongWritable for
+ * LONG-category types, OrcTimestamp for the timestamp categories).
+ *
+ * @param setLong callback into the row's setLong (used by primitive
Long-backed types)
+ * @param set callback into the row's generic set (used by object-backed
types, e.g. nanos)
+ */
+ def makeDeserializer(
+ setLong: (Int, Long) => Unit,
+ set: (Int, Any) => Unit): (Int, WritableComparable[_]) => Unit
+
+ // ==================== Predicate Pushdown ====================
+
+ /**
+ * The ORC PredicateLeaf.Type for this type, consumed by
OrcFilters.getPredicateLeafType. Return
+ * Some(type) to enable predicate pushdown for this type.
+ *
+ * A None keeps the pre-existing OrcFilters behavior for a type with no leaf
mapping: it reaches
+ * getPredicateLeafType's final arm, which throws
unsupportedOperationForDataTypeError. In
+ * practice that arm is only reached for a column already deemed
pushdown-eligible upstream
+ * (OrcFiltersBase.getSearchableTypeMap admits any AtomicType), so a
framework AtomicType that
+ * returns None and is used in a pushed filter would fail during planning. A
future type that
+ * must be pushdown-eligible therefore has to return Some here (and, if its
literal needs
+ * conversion, override castFilterLiteral); returning None is only safe for
a type that can never
+ * reach getPredicateLeafType.
+ *
+ * TimeType returns Some(LONG). The nanosecond-timestamp types return None:
they had no
+ * getPredicateLeafType arm before this change either, so the behavior
(throw if ever reached) is
+ * preserved, not newly introduced.
+ */
+ def predicateLeafType: Option[PredicateLeaf.Type] = None
+
+ /**
+ * Casts a filter literal to the ORC search-argument representation
(OrcFilters.castLiteralValue).
+ * Only meaningful for types that also return a predicateLeafType. Default
is identity.
+ */
+ def castFilterLiteral(value: Any): Any = value
+}
+
+/**
+ * Factory object for creating OrcTypeOps instances.
+ *
+ * Provides forward lookup (DataType -> ops) for the framework-type dispatch
arms at ORC
+ * integration sites. apply() returns Some only for framework-managed types,
so callers fall back
+ * to the original inline path for everything else.
+ */
+private[orc] object OrcTypeOps {
+
+ /**
+ * Returns an OrcTypeOps instance for the given DataType, if supported.
+ *
+ * Returns None if the type has no ORC ops. This is the single registration
point for all ORC
+ * type operations.
+ */
+ def apply(dt: DataType): Option[OrcTypeOps] = dt match {
+ case tt: TimeType => Some(TimeTypeOrcOps(tt))
+ case t: TimestampLTZNanosType => Some(TimestampLTZNanosOrcOps(t))
+ case t: TimestampNTZNanosType => Some(TimestampNTZNanosOrcOps(t))
+ // Add new types here - single registration point
+ case _ => None
+ }
+
+ /**
+ * Extractor so a `dataType match` arm can bind the ops in a single lookup:
+ * {{{
+ * case OrcTypeOps(ops) => ops.method(...)
+ * }}}
+ * Delegates to apply(); returning the same Option keeps the registry the
single source of truth
+ * and avoids the double lookup of a `case _ if OrcTypeOps(dt).isDefined =>
OrcTypeOps(dt).get`
+ * guard.
+ */
+ def unapply(dt: DataType): Option[OrcTypeOps] = apply(dt)
+}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/TimeTypeOrcOps.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/TimeTypeOrcOps.scala
new file mode 100644
index 000000000000..1896a2712eec
--- /dev/null
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/TimeTypeOrcOps.scala
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc.types.ops
+
+import java.time.LocalTime
+
+import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf
+import org.apache.hadoop.io.{LongWritable, WritableComparable}
+import org.apache.orc.TypeDescription
+
+import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.localTimeToNanos
+import org.apache.spark.sql.types.{LongType, TimeType}
+
+/**
+ * ORC operations for TimeType.
+ *
+ * TimeType is a primitive Long-backed type (nanoseconds since midnight). ORC
has no TIME category,
+ * so it is stored in the LONG physical category with the true Spark type
recovered on read from
+ * the CATALYST_TYPE_ATTRIBUTE_NAME attribute. Read and write are pure Long
pass-through: the
+ * internal nanos-of-day value IS the stored LONG (no unit conversion, unlike
Parquet which may
+ * store MICROS). Precision affects only display, so nothing is truncated here.
+ *
+ * @param t the TimeType (precision is unused on the ORC path; kept for
symmetry with the other
+ * storage ops and possible future precision-aware behavior).
+ * @since 5.0.0
+ */
+case class TimeTypeOrcOps(t: TimeType) extends OrcTypeOps {
+
+ // ==================== Schema Mapping ====================
+
+ // Was: OrcUtils.getOrcSchemaString `case ... | _: TimeType =>
LongType.catalogString`
+ override def orcSchemaString: String = LongType.catalogString
+
+ // Was: OrcUtils.orcTypeDescription `case tm: TimeType => new
TypeDescription(LONG) ...`
+ override def orcCategory: TypeDescription.Category =
TypeDescription.Category.LONG
+
+ // ==================== Value Write (serialize) ====================
+
+ // Was: OrcSerializer.newConverter `case ... | _: TimeType => ...
LongWritable ...`
+ override def makeSerializer(
+ reuseObj: Boolean): (SpecializedGetters, Int) => WritableComparable[_] =
+ if (reuseObj) {
+ val result = new LongWritable()
+ (getter: SpecializedGetters, ordinal: Int) => {
+ result.set(getter.getLong(ordinal))
+ result
+ }
+ } else {
+ (getter: SpecializedGetters, ordinal: Int) => new
LongWritable(getter.getLong(ordinal))
+ }
+
+ // ==================== Row-Based Read (deserialize) ====================
+
+ // Was: OrcDeserializer.newWriter `case ... | _: TimeType => ... setLong
...`
+ override def makeDeserializer(
+ setLong: (Int, Long) => Unit,
+ set: (Int, Any) => Unit): (Int, WritableComparable[_]) => Unit =
+ (ordinal: Int, value: WritableComparable[_]) =>
+ setLong(ordinal, value.asInstanceOf[LongWritable].get)
+
+ // ==================== Predicate Pushdown ====================
+
+ // Was: OrcFilters.getPredicateLeafType `case ... | _: TimeType =>
PredicateLeaf.Type.LONG`
+ override def predicateLeafType: Option[PredicateLeaf.Type] =
Some(PredicateLeaf.Type.LONG)
+
+ // Was: OrcFilters.castLiteralValue `case _: TimeType if
value.isInstanceOf[LocalTime] => ...`
+ override def castFilterLiteral(value: Any): Any = value match {
+ case lt: LocalTime => localTimeToNanos(lt)
+ case other => other
+ }
+}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/TimestampNanosOrcOps.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/TimestampNanosOrcOps.scala
new file mode 100644
index 000000000000..87a4bd252a88
--- /dev/null
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/types/ops/TimestampNanosOrcOps.scala
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc.types.ops
+
+import org.apache.hadoop.io.WritableComparable
+import org.apache.orc.TypeDescription
+import org.apache.orc.mapred.OrcTimestamp
+
+import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.types.{TimestampLTZNanosType,
TimestampNTZNanosType}
+import org.apache.spark.unsafe.types.TimestampNanosVal
+
+/**
+ * ORC operations for TimestampLTZNanosType (nanosecond precision, with time
zone).
+ *
+ * Stored in ORC as the TIMESTAMP_INSTANT category (matching TIMESTAMP with
local time zone). The
+ * internal TimestampNanosVal is converted through java.time.Instant in both
directions; the true
+ * Spark type and precision are recovered on read from the
CATALYST_TYPE_ATTRIBUTE_NAME attribute.
+ * ORC does not reuse the OrcTimestamp object (the conversion is already
expensive), so the
+ * serializer runs with reuse disabled regardless of the reuseObj hint.
+ *
+ * No predicate pushdown: the inline OrcFilters code never added a nanos arm,
so predicateLeafType
+ * stays None (inherited default).
+ *
+ * @since 5.0.0
+ */
+case class TimestampLTZNanosOrcOps(t: TimestampLTZNanosType) extends
OrcTypeOps {
+
+ // Was: OrcUtils.getOrcSchemaString
+ // `case _: TimestampLTZNanosType => "timestamp with local time zone"`
+ override def orcSchemaString: String = "timestamp with local time zone"
+
+ // Was: OrcUtils.orcTypeDescription `case t: TimestampLTZNanosType => ...
TIMESTAMP_INSTANT ...`
+ override def orcCategory: TypeDescription.Category =
TypeDescription.Category.TIMESTAMP_INSTANT
+
+ // Was: OrcSerializer.newConverter `case t: TimestampLTZNanosType => ...`
+ override def makeSerializer(
+ reuseObj: Boolean): (SpecializedGetters, Int) => WritableComparable[_] =
+ (getter: SpecializedGetters, ordinal: Int) => {
+ val v = getter.get(ordinal, t).asInstanceOf[TimestampNanosVal]
+ val instant = DateTimeUtils.timestampNanosToInstant(v)
+ val result = new OrcTimestamp(instant.toEpochMilli)
+ result.setNanos(instant.getNano)
+ result
+ }
+
+ // Was: OrcDeserializer.newWriter `case t: TimestampLTZNanosType => ...`
+ override def makeDeserializer(
+ setLong: (Int, Long) => Unit,
+ set: (Int, Any) => Unit): (Int, WritableComparable[_]) => Unit =
+ (ordinal: Int, value: WritableComparable[_]) => {
+ val ts = value.asInstanceOf[OrcTimestamp]
+ val instant = ts.toInstant
+ set(ordinal, DateTimeUtils.instantToTimestampNanos(instant, t.precision))
+ }
+}
+
+/**
+ * ORC operations for TimestampNTZNanosType (nanosecond precision, without
time zone).
+ *
+ * Stored in ORC as the TIMESTAMP category. The internal TimestampNanosVal is
converted through
+ * java.time.LocalDateTime in both directions; the true Spark type and
precision are recovered on
+ * read from the CATALYST_TYPE_ATTRIBUTE_NAME attribute. As with the LTZ
variant, the OrcTimestamp
+ * object is not reused.
+ *
+ * No predicate pushdown (predicateLeafType stays None).
+ *
+ * @since 5.0.0
+ */
+case class TimestampNTZNanosOrcOps(t: TimestampNTZNanosType) extends
OrcTypeOps {
+
+ // Was: OrcUtils.getOrcSchemaString `case _: TimestampNTZNanosType =>
"timestamp"`
+ override def orcSchemaString: String = "timestamp"
+
+ // Was: OrcUtils.orcTypeDescription `case t: TimestampNTZNanosType => ...
TIMESTAMP ...`
+ override def orcCategory: TypeDescription.Category =
TypeDescription.Category.TIMESTAMP
+
+ // Was: OrcSerializer.newConverter `case t: TimestampNTZNanosType => ...`
+ override def makeSerializer(
+ reuseObj: Boolean): (SpecializedGetters, Int) => WritableComparable[_] =
+ (getter: SpecializedGetters, ordinal: Int) => {
+ val v = getter.get(ordinal, t).asInstanceOf[TimestampNanosVal]
+ val localDateTime = DateTimeUtils.timestampNanosToLocalDateTime(v)
+ val ts = java.sql.Timestamp.valueOf(localDateTime)
+ val result = new OrcTimestamp(ts.getTime)
+ result.setNanos(ts.getNanos)
+ result
+ }
+
+ // Was: OrcDeserializer.newWriter `case t: TimestampNTZNanosType => ...`
+ override def makeDeserializer(
+ setLong: (Int, Long) => Unit,
+ set: (Int, Any) => Unit): (Int, WritableComparable[_]) => Unit =
+ (ordinal: Int, value: WritableComparable[_]) => {
+ val ts = value.asInstanceOf[OrcTimestamp]
+ val localDateTime = ts.toLocalDateTime
+ set(ordinal, DateTimeUtils.localDateTimeToTimestampNanos(localDateTime,
t.precision))
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]