This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 1c4903dc29 [spark] Fix a shifted date written from before the
Gregorian cutover (#10160)
1c4903dc29 is described below
commit 1c4903dc29ecd4dd6fa3b2c8bebaee6711115d92
Author: cxzl25 <[email protected]>
AuthorDate: Fri Sep 25 21:10:36 2026 +0800
[spark] Fix a shifted date written from before the Gregorian cutover
(#10160)
---
.../java/org/apache/paimon/spark/SparkRow.java | 6 ++++-
.../org/apache/paimon/spark/sql/DDLTestBase.scala | 27 ++++++++++++++++++++++
2 files changed, 32 insertions(+), 1 deletion(-)
diff --git
a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
index 297ec24f60..19d5962d2e 100644
---
a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
+++
b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkRow.java
@@ -208,7 +208,11 @@ public class SparkRow implements InternalRow, Serializable
{
private static int toPaimonDate(Object object) {
if (object instanceof Date) {
- return DateTimeUtils.toInternal((Date) object);
+ // Spark builds this java.sql.Date from its internal day through
the hybrid calendar,
+ // which is Julian before 1582-10-15, so only Spark's own inverse
recovers the day.
+ // Counting epoch days off the instant follows the java.time rules
instead, which
+ // leaves a date from before the cutover stored days away from the
one written.
+ return
org.apache.spark.sql.catalyst.util.DateTimeUtils.fromJavaDate((Date) object);
} else {
return DateTimeUtils.toInternal((LocalDate) object);
}
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala
index c21ed067b5..a81d9dad9f 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala
@@ -928,6 +928,33 @@ abstract class DDLTestBase extends PaimonSparkTestBase {
}
}
+ test("Paimon DDL: write a date from before the Gregorian cutover") {
+ Seq(true, false).foreach {
+ datetimeJava8APIEnabled =>
+ withSparkSQLConf("spark.sql.datetime.java8API.enabled" ->
datetimeJava8APIEnabled.toString) {
+ withTable("paimon_tbl") {
+ sql("CREATE TABLE paimon_tbl (id INT, dt DATE) USING paimon")
+ sql("INSERT INTO paimon_tbl VALUES (1, date'1000-01-01')")
+ sql("INSERT INTO paimon_tbl VALUES (2, date'1582-10-04')")
+ sql("INSERT INTO paimon_tbl VALUES (3, date'1582-10-15')")
+ sql("INSERT INTO paimon_tbl VALUES (4, date'1970-01-01')")
+
+ checkAnswer(
+ sql("SELECT id, cast(dt as string) FROM paimon_tbl ORDER BY id"),
+ Row(1, "1000-01-01") :: Row(2, "1582-10-04") :: Row(3,
"1582-10-15") ::
+ Row(4, "1970-01-01") :: Nil
+ )
+
+ checkAnswer(sql("SELECT id FROM paimon_tbl WHERE dt =
date'1582-10-04'"), Row(2) :: Nil)
+
+ checkAnswer(
+ sql("SELECT id FROM paimon_tbl WHERE dt < date'1582-10-15' ORDER
BY id"),
+ Row(1) :: Row(2) :: Nil)
+ }
+ }
+ }
+ }
+
test("Paimon DDL: select table with timestamp and timestamp_ntz with
filter") {
Seq(true, false).foreach {
datetimeJava8APIEnabled =>
