This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 18df87b9f5b5 [SPARK-57813][SQL] Support nanosecond-precision timestamp
types as file-source partition columns
18df87b9f5b5 is described below
commit 18df87b9f5b56dafc69394e3673cf37071954333
Author: Stevo Mitric <[email protected]>
AuthorDate: Thu Jul 9 20:31:18 2026 +0800
[SPARK-57813][SQL] Support nanosecond-precision timestamp types as
file-source partition columns
### What changes were proposed in this pull request?
Widen the timestamp guard in `PartitioningUtils.castPartValueToDesiredType`
so that partition values are cast when the desired partition-column type is an
`AnyTimestampNanoType` (`TimestampNTZNanosType`/`TimestampLTZNanosType` at
precision 7-9)
### Why are the changes needed?
Without this, dynamically writing then reading back a table partitioned on
a nanosecond-precision timestamp column failed to reconstruct the partition
value and raised `INVALID_PARTITION_VALUE` when casting the escaped directory
name to the declared nanos type.
### Does this PR introduce _any_ user-facing change?
Yes. Nanosecond-precision timestamp columns
(`TIMESTAMP_NTZ(p)`/`TIMESTAMP_LTZ(p)`, p in [7, 9]) can now be used as
file-source partition columns.
Note: nanos partition values are reconstructed only when an explicit reader
schema is supplied; partition-type inference yields only micros
`TimestampType`/`TimestampNTZType`, never a nanos type (parallels `TIME(p)`).
### How was this patch tested?
Added `PartitionedWriteSuite` coverage: write-then-read round-trip for both
NTZ and LTZ nanos partition columns.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 4.8
Closes #57082 from stevomitric/stevomitric/spark-57813-nanos.
Authored-by: Stevo Mitric <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../execution/datasources/PartitioningUtils.scala | 2 +-
.../spark/sql/sources/PartitionedWriteSuite.scala | 63 ++++++++++++++++++++++
2 files changed, 64 insertions(+), 1 deletion(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
index 3d870c968cd6..83a094c250d6 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
@@ -562,7 +562,7 @@ object PartitioningUtils extends SQLConfHelper {
Cast(Literal(value), DateType, Some(zoneId.getId)).eval()
case tt: TimeType => Cast(Literal(unescapePathName(value)), tt).eval()
// Timestamp types
- case dt if AnyTimestampType.acceptsType(dt) =>
+ case dt if AnyTimestampType.acceptsType(dt) ||
AnyTimestampNanoType.acceptsType(dt) =>
Try {
Cast(Literal(unescapePathName(value)), dt, Some(zoneId.getId)).eval()
}.getOrElse {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
index fb37ffaa7aa2..6154687a1407 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
@@ -254,6 +254,69 @@ class PartitionedWriteSuite extends SharedSparkSession {
}
}
}
+
+ test("SPARK-57813: Dynamic writes/reads of nanosecond TIMESTAMP_NTZ
partitions") {
+ Seq(
+ "2019-01-01 12:00:00.0000019" -> TimestampNTZNanosType(7),
+ "2019-01-01 12:34:56.12345678" -> TimestampNTZNanosType(8),
+ "2019-01-01 23:59:59.999999999" -> TimestampNTZNanosType(9)
+ ).foreach { case (tsStr, tsType) =>
+ withTempPath { f =>
+ val df = sql(s"select 0 AS id, cast('$tsStr' as ${tsType.sql}) AS tt")
+ assert(df.schema("tt").dataType === tsType)
+ df.write
+ .partitionBy("tt")
+ .format("parquet")
+ .save(f.getAbsolutePath)
+ val files =
TestUtils.recursiveList(f).filter(_.getAbsolutePath.endsWith("parquet"))
+ assert(files.length == 1)
+ checkPartitionValues(files.head, tsStr)
+ val schema = new StructType()
+ .add("id", IntegerType)
+ .add("tt", tsType)
+ val read =
spark.read.schema(schema).format("parquet").load(f.getAbsolutePath)
+ checkAnswer(read, df)
+ // pruning on the nanos partition value: the matching predicate keeps
the row
+ checkAnswer(read.where(s"tt = cast('$tsStr' as ${tsType.sql})"), df)
+ // and a deliberately non-matching predicate excludes the partition
entirely
+ checkAnswer(
+ read.where(s"tt = cast('2020-02-02 02:02:02' as ${tsType.sql})"),
Seq.empty[Row])
+ }
+ }
+ }
+
+ test("SPARK-57813: Dynamic writes/reads of nanosecond TIMESTAMP_LTZ
partitions") {
+ // The LTZ write directory string depends on the session zone, so pin it
for determinism.
+ withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
+ Seq(
+ "2019-01-01 12:00:00.0000019" -> TimestampLTZNanosType(7),
+ "2019-01-01 12:34:56.12345678" -> TimestampLTZNanosType(8),
+ "2019-01-01 23:59:59.999999999" -> TimestampLTZNanosType(9)
+ ).foreach { case (tsStr, tsType) =>
+ withTempPath { f =>
+ val df = sql(s"select 0 AS id, cast('$tsStr' as ${tsType.sql}) AS
tt")
+ assert(df.schema("tt").dataType === tsType)
+ df.write
+ .partitionBy("tt")
+ .format("parquet")
+ .save(f.getAbsolutePath)
+ val files =
TestUtils.recursiveList(f).filter(_.getAbsolutePath.endsWith("parquet"))
+ assert(files.length == 1)
+ checkPartitionValues(files.head, tsStr)
+ val schema = new StructType()
+ .add("id", IntegerType)
+ .add("tt", tsType)
+ val read =
spark.read.schema(schema).format("parquet").load(f.getAbsolutePath)
+ checkAnswer(read, df)
+ // pruning on the nanos partition value: the matching predicate
keeps the row
+ checkAnswer(read.where(s"tt = cast('$tsStr' as ${tsType.sql})"), df)
+ // and a deliberately non-matching predicate excludes the partition
entirely
+ checkAnswer(
+ read.where(s"tt = cast('2020-02-02 02:02:02' as ${tsType.sql})"),
Seq.empty[Row])
+ }
+ }
+ }
+ }
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
