This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 94b9668444 [spark] Fix a shifted timestamp written from before a zone
left local mean time (#10117)
94b9668444 is described below
commit 94b96684441ba7fe0690a007a3e48f3dbaa852cf
Author: cxzl25 <[email protected]>
AuthorDate: Thu Sep 24 13:09:01 2026 +0800
[spark] Fix a shifted timestamp written from before a zone left local mean
time (#10117)
---
.../java/org/apache/paimon/spark/SparkRow.java | 9 +++++-
.../org/apache/paimon/spark/sql/DDLTestBase.scala | 36 ++++++++++++++++++++++
2 files changed, 44 insertions(+), 1 deletion(-)
diff --git
a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
index a8e6ae143b..297ec24f60 100644
---
a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
+++
b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
@@ -220,7 +220,14 @@ public class SparkRow implements InternalRow, Serializable
{
if (TypeUtils.treatPaimonTimestampTypeAsSparkTimestampType()) {
return Timestamp.fromSQLTimestamp(ts);
} else {
- return Timestamp.fromInstant(ts.toInstant());
+ // Spark builds this java.sql.Timestamp from its internal
micros through the
+ // hybrid calendar and the legacy time zone rules, so only
Spark's own inverse
+ // recovers the instant. Instant#toInstant applies the
java.time rules instead,
+ // which disagree wherever the two differ - a zone's pre-1900
offset was rarely a
+ // whole number of hours (Asia/Shanghai was +08:05:43 until
1901), and the hybrid
+ // calendar is Julian before 1582. The value would be stored
shifted.
+ return Timestamp.fromMicros(
+
org.apache.spark.sql.catalyst.util.DateTimeUtils.fromJavaTimestamp(ts));
}
} else if (object instanceof java.time.Instant) {
Instant instant = (Instant) object;
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala
index 70e38deecd..c21ed067b5 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala
@@ -892,6 +892,42 @@ abstract class DDLTestBase extends PaimonSparkTestBase {
}
}
+ test("Paimon DDL: write a timestamp from before the zone left local mean
time") {
+ // Spark hands out a java.sql.Timestamp built through the hybrid calendar
and the legacy
+ // time zone rules. Recovering the instant with java.time rules instead
shifts any value
+ // the two disagree on: Asia/Shanghai ran on +08:05:43 until 1901 and
Europe/Paris on
+ // +00:09:21 until 1911, so such a timestamp used to be stored minutes
away from the value
+ // that was written, and then neither matched an equality filter nor
stayed out of the
+ // rows a greater-than filter returns.
+ // Only the java.sql.Timestamp hand-off is affected, so pin the flag that
selects it rather
+ // than relying on its default. The spark-sql CLI turns it on at startup
(SPARK-31893), which
+ // is why the shift never shows up there.
+ withSparkSQLConf("spark.sql.datetime.java8API.enabled" -> "false") {
+ Seq("Asia/Shanghai", "Europe/Paris", "UTC").foreach {
+ zone =>
+ withTimeZone(zone) {
+ withTable("paimon_tbl") {
+ sql("CREATE TABLE paimon_tbl (id INT, ts TIMESTAMP) USING
paimon")
+ sql("INSERT INTO paimon_tbl VALUES (1, timestamp'1900-01-01
00:00:00')")
+ sql("INSERT INTO paimon_tbl VALUES (2, timestamp'1970-01-01
00:00:00')")
+
+ checkAnswer(
+ sql("SELECT id, cast(ts as string) FROM paimon_tbl ORDER BY
id"),
+ Row(1, "1900-01-01 00:00:00") :: Row(2, "1970-01-01 00:00:00")
:: Nil)
+
+ checkAnswer(
+ sql("SELECT id FROM paimon_tbl WHERE ts = timestamp'1900-01-01
00:00:00'"),
+ Row(1) :: Nil)
+
+ checkAnswer(
+ sql("SELECT id FROM paimon_tbl WHERE ts > timestamp'1900-01-01
00:00:00'"),
+ Row(2) :: Nil)
+ }
+ }
+ }
+ }
+ }
+
test("Paimon DDL: select table with timestamp and timestamp_ntz with
filter") {
Seq(true, false).foreach {
datetimeJava8APIEnabled =>
