This is an automated email from the ASF dual-hosted git repository.
MaxGekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9a2bb2a3cea3 [SPARK-55444][SQL] Remove dead TimeType branches from the
Parquet *Default methods
9a2bb2a3cea3 is described below
commit 9a2bb2a3cea3f31680d163f6025992cd64b36378
Author: Stevo Mitric <[email protected]>
AuthorDate: Tue Jun 23 11:03:15 2026 +0200
[SPARK-55444][SQL] Remove dead TimeType branches from the Parquet *Default
methods
### What changes were proposed in this pull request?
Phase 3a ([SPARK-55444](https://issues.apache.org/jira/browse/SPARK-55444))
routes `TimeType` through the Types Framework first
(`ParquetTypeOps(dt).map(...).getOrElse(...Default)`), so the `case _:
TimeType` arms left in the `*Default` fallbacks are unreachable. This removes
them from `ParquetSchemaConverter.convertFieldDefault`,
`ParquetWriteSupport.makeWriterDefault`, and
`ParquetRowConverter.newConverterDefault` .
### Why are the changes needed?
Dead code: each `*Default` method has a single caller (the framework
`getOrElse`), and `ParquetTypeOps` always handles `TimeType`, so those arms
never run. Mirrors SPARK-57372's follow-up (`f3d1de5ced5`), which removed the
analogous dead `TimeType` paths in catalyst/api/connect/hive but did not touch
these Parquet ones.
### Does this PR introduce _any_ user-facing change?
No. Pure dead-code removal; `TimeType` is handled by the framework path.
### How was this patch tested?
Existing suites, no behavior change: `TimeTypeParquetOpsSuite` +
`ParquetSchemaSuite` (145 tests) and `ParquetIOSuite` TimeType reads (2 tests).
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 4.8)
Closes #56656 from stevomitric/stevomitric/parquet-tf-deadcode-cleanup.
Authored-by: Stevo Mitric <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
---
.../datasources/parquet/ParquetRowConverter.scala | 16 ----------------
.../datasources/parquet/ParquetSchemaConverter.scala | 6 ------
.../datasources/parquet/ParquetWriteSupport.scala | 9 ---------
3 files changed, 31 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
index 2200179f5a9e..5c41f30255a4 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
@@ -526,22 +526,6 @@ private[parquet] class ParquetRowConverter(
}
}
- case t: TimeType
- if
parquetType.getLogicalTypeAnnotation.isInstanceOf[TimeLogicalTypeAnnotation] &&
{
- val unit = parquetType.getLogicalTypeAnnotation
- .asInstanceOf[TimeLogicalTypeAnnotation].getUnit
- unit == TimeUnit.MICROS || unit == TimeUnit.NANOS
- } =>
- val fileStoresNanos = parquetType.getLogicalTypeAnnotation
- .asInstanceOf[TimeLogicalTypeAnnotation].getUnit == TimeUnit.NANOS
- val precision = t.precision
- new ParquetPrimitiveConverter(updater) {
- override def addLong(value: Long): Unit = {
- val nanos = if (fileStoresNanos) value else
DateTimeUtils.microsToNanos(value)
- this.updater.setLong(DateTimeUtils.truncateTimeToPrecision(nanos,
precision))
- }
- }
-
// A repeated field that is neither contained by a `LIST`- or
`MAP`-annotated group nor
// annotated by `LIST` or `MAP` should be interpreted as a required list
of required
// elements where the element type is the type of the field.
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
index a0acc40f9e01..df9db863f85f 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala
@@ -713,12 +713,6 @@ class SparkToParquetSchemaConverter(
Types.primitive(INT32, repetition)
.as(LogicalTypeAnnotation.dateType()).named(field.name)
- case t: TimeType =>
- // Precision 0..6 is stored as TIME(MICROS); precision 7..9 as
TIME(NANOS).
- val unit = if (t.precision > TimeType.MICROS_PRECISION) TimeUnit.NANOS
else TimeUnit.MICROS
- Types.primitive(INT64, repetition)
- .as(LogicalTypeAnnotation.timeType(false, unit)).named(field.name)
-
// NOTE: Spark SQL can write timestamp values to Parquet using INT96,
TIMESTAMP_MICROS or
// TIMESTAMP_MILLIS. TIMESTAMP_MICROS is recommended but INT96 is the
default to keep the
// behavior same as before.
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
index c7d7426a94ea..48e57f1d6bd3 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
@@ -307,15 +307,6 @@ class ParquetWriteSupport extends
WriteSupport[InternalRow] with Logging {
recordConsumer.addLong(
timestampNanosToEpochNanos(row.getTimestampNTZNanos(ordinal),
isNtz = true))
- case t: TimeType if t.precision > TimeType.MICROS_PRECISION =>
- // Precision 7..9 is stored as TIME(NANOS); internal storage is
already nanos.
- (row: SpecializedGetters, ordinal: Int) =>
- recordConsumer.addLong(row.getLong(ordinal))
-
- case _: TimeType =>
- (row: SpecializedGetters, ordinal: Int) =>
-
recordConsumer.addLong(DateTimeUtils.nanosToMicros(row.getLong(ordinal)))
-
case BinaryType =>
(row: SpecializedGetters, ordinal: Int) =>
recordConsumer.addBinary(Binary.fromReusedByteArray(row.getBinary(ordinal)))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
