This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8ca77e0945d9 [SPARK-58742][SQL][TEST] Add nanosecond-timestamp
coverage for collect_set
8ca77e0945d9 is described below
commit 8ca77e0945d9299d697b184914223ee36cb0b604
Author: Stevo Mitric <[email protected]>
AuthorDate: Sat Aug 15 10:26:34 2026 +0000
[SPARK-58742][SQL][TEST] Add nanosecond-timestamp coverage for collect_set
### What changes were proposed in this pull request?
Add an end-to-end test to `TimestampNanosFunctionsSuiteBase` (run in both
ANSI modes) asserting that collect_set deduplicates on the full nanosecond
value: sub-microsecond-distinct values stay distinct at p=9 but collapse when
floored below the grid at p in [7, 8]. Also add a deterministic golden SQL case
(`sort_array(collect_set(...))`) to `timestamp-ntz-nanos.sql` /
`timestamp-ltz-nanos.sql`.
### Why are the changes needed?
Extend test coverage for timestamp nanosecond precision datatype.
### Does this PR introduce _any_ user-facing change?
No, test only change.
### How was this patch tested?
Test-only change.
### Was this patch authored or co-authored using generative AI tooling?
Co-Authored-By: Claude Code 4.8
Closes #57965 from stevomitric/stevomitric/nanos-collect-set-tests.
Authored-by: Stevo Mitric <[email protected]>
Signed-off-by: Uros Bojanic <[email protected]>
---
.../analyzer-results/timestamp-ltz-nanos.sql.out | 11 +++++++
.../analyzer-results/timestamp-ntz-nanos.sql.out | 11 +++++++
.../sql-tests/inputs/timestamp-ltz-nanos.sql | 10 +++++++
.../sql-tests/inputs/timestamp-ntz-nanos.sql | 9 ++++++
.../sql-tests/results/timestamp-ltz-nanos.sql.out | 11 +++++++
.../sql-tests/results/timestamp-ntz-nanos.sql.out | 11 +++++++
.../sql/TimestampNanosFunctionsSuiteBase.scala | 34 ++++++++++++++++++++++
7 files changed, 97 insertions(+)
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out
index 93c3ef991b96..fd2179dd3cdf 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out
@@ -751,6 +751,17 @@ Aggregate [mode(c#x, 0, 0, None) AS mode(c)#x]
+- LocalRelation [c#x]
+-- !query
+SELECT sort_array(collect_set(c)) FROM VALUES
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'),
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'),
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC') AS t(c)
+-- !query analysis
+Aggregate [sort_array(collect_set(c#x, 0, 0, true), true) AS
sort_array(collect_set(c), true)#x]
++- SubqueryAlias t
+ +- LocalRelation [c#x]
+
+
-- !query
SELECT unix_timestamp(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789')
-- !query analysis
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out
index 02484a99b4b7..5445f457e3ab 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out
@@ -679,6 +679,17 @@ Aggregate [mode(c#x, 0, 0, None) AS mode(c)#x]
+- LocalRelation [c#x]
+-- !query
+SELECT sort_array(collect_set(c)) FROM VALUES
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'),
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'),
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') AS t(c)
+-- !query analysis
+Aggregate [sort_array(collect_set(c#x, 0, 0, true), true) AS
sort_array(collect_set(c), true)#x]
++- SubqueryAlias t
+ +- LocalRelation [c#x]
+
+
-- !query
SELECT unix_timestamp(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789')
-- !query analysis
diff --git
a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql
b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql
index 9a172a2fe2ae..1840ca375ea4 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql
@@ -229,6 +229,16 @@ SELECT mode(c) FROM VALUES
(TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'),
(TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC') AS t(c);
+-- SPARK-56822: collect_set over nanosecond-precision TIMESTAMP_LTZ. It
deduplicates on the full
+-- sub-microsecond value: the two .000000001 rows collapse to one, the
.000000999 row stays, so the
+-- sorted set has two distinct elements and the element type stays
TIMESTAMP_LTZ(9); values render
+-- in the session time zone (America/Los_Angeles). collect_set order is
non-deterministic, so the
+-- output is stabilized with sort_array.
+SELECT sort_array(collect_set(c)) FROM VALUES
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'),
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'),
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC') AS t(c);
+
-- SPARK-57528: unix_timestamp / to_unix_timestamp over nanosecond-precision
values. The result is
-- whole-second BIGINT; the sub-second digits are dropped. A literal without
an explicit zone is
-- read in the session time zone (America/Los_Angeles, UTC-08:00); an
explicit-zone literal fixes
diff --git
a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql
b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql
index 2a571619cf55..7fb6fdcbc019 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql
@@ -201,6 +201,15 @@ SELECT mode(c) FROM VALUES
(TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'),
(TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') AS t(c);
+-- SPARK-56822: collect_set over nanosecond-precision TIMESTAMP_NTZ. It
deduplicates on the full
+-- sub-microsecond value: the two .000000001 rows collapse to one, the
.000000999 row stays, so the
+-- sorted set has two distinct elements and the element type stays
TIMESTAMP_NTZ(9). collect_set
+-- order is non-deterministic, so the output is stabilized with sort_array.
+SELECT sort_array(collect_set(c)) FROM VALUES
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'),
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'),
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') AS t(c);
+
-- SPARK-57528: unix_timestamp / to_unix_timestamp over nanosecond-precision
values. The result is
-- whole-second BIGINT; the sub-second digits are dropped and NTZ applies no
zone shift, so the
-- wall-clock value is read as the epoch instant.
diff --git
a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out
b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out
index 4c036133a551..a28831c6b7d8 100644
--- a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out
@@ -843,6 +843,17 @@ struct<mode(c):timestamp_ltz(9)>
2019-12-31 16:00:00.000000001
+-- !query
+SELECT sort_array(collect_set(c)) FROM VALUES
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'),
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'),
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC') AS t(c)
+-- !query schema
+struct<sort_array(collect_set(c), true):array<timestamp_ltz(9)>>
+-- !query output
+[2019-12-31 16:00:00.000000001,2019-12-31 16:00:00.000000999]
+
+
-- !query
SELECT unix_timestamp(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789')
-- !query schema
diff --git
a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out
b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out
index e10a25456268..bbec01467ac2 100644
--- a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out
@@ -761,6 +761,17 @@ struct<mode(c):timestamp_ntz(9)>
2020-01-01 00:00:00.000000001
+-- !query
+SELECT sort_array(collect_set(c)) FROM VALUES
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'),
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'),
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') AS t(c)
+-- !query schema
+struct<sort_array(collect_set(c), true):array<timestamp_ntz(9)>>
+-- !query output
+[2020-01-01 00:00:00.000000001,2020-01-01 00:00:00.000000999]
+
+
-- !query
SELECT unix_timestamp(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789')
-- !query schema
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala
b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala
index 20a140491b2f..f6f3d105d462 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala
@@ -740,6 +740,40 @@ abstract class TimestampNanosFunctionsSuiteBase extends
SharedSparkSession {
}
}
+ // collect_set over nanosecond-precision timestamps (SPARK-56822).
`CollectSet` deduplicates via a
+ // `HashSet` keyed on the physical `TimestampNanosVal`, whose
`equals`/`hashCode` cover the full
+ // `(epochMicros, nanosWithinMicro)` pair, so sub-microsecond-distinct
values are kept distinct;
+ // the result element type is exactly `child.dataType`.
+
+ test("SPARK-56822: collect_set over nanos deduplicates on the full
sub-microsecond value") {
+ // Two values share the microsecond and differ only in the last nanosecond
digit. Flooring the
+ // input to precision `p` collapses them when p < 9 (the distinguishing
digit is below the
+ // grid), but keeps them apart at p = 9. collect_set must dedup on the
full stored value,
+ // not on micros.
+ Seq(7, 8, 9).foreach { p =>
+ val schema = new StructType().add("ntz", TimestampNTZNanosType(p))
+ val data = Seq(
+ Row(LocalDateTime.parse("2020-01-01T12:34:56.123456780")),
+ Row(LocalDateTime.parse("2020-01-01T12:34:56.123456789")),
+ Row(LocalDateTime.parse("2020-01-01T12:34:56.123456780")))
+ val df = spark.createDataFrame(spark.sparkContext.parallelize(data),
schema)
+
+ val res = df.selectExpr("collect_set(ntz)")
+ assert(res.schema.head.dataType === ArrayType(TimestampNTZNanosType(p),
containsNull = false))
+
+ // Values are floored to `p` on ingestion, so the distinct set depends
on `p`.
+ val expected = p match {
+ case 7 => Set(LocalDateTime.parse("2020-01-01T12:34:56.123456700"))
+ case 8 => Set(LocalDateTime.parse("2020-01-01T12:34:56.123456780"))
+ case _ => Set(
+ LocalDateTime.parse("2020-01-01T12:34:56.123456780"),
+ LocalDateTime.parse("2020-01-01T12:34:56.123456789"))
+ }
+ val collected = res.collect().head.getSeq[LocalDateTime](0).toSet
+ assert(collected === expected, s"collect_set(p=$p) expected $expected,
got $collected")
+ }
+ }
+
test("SPARK-57816: date_format / to_char / to_varchar over
nanosecond-precision timestamps") {
// The 9-`S` pattern is a fixed-width fraction field, so it always emits 9
digits; truncating to
// precision `p` zeros the low digits (floor); it does not drop them. The
session zone is
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
