This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new df2fddd5d008 [SPARK-47473][SQL] Fix correctness issue of converting
postgres INFINITY timestamps
df2fddd5d008 is described below
commit df2fddd5d00860506534d5302f9b3f944d775deb
Author: Kent Yao <y...@apache.org>
AuthorDate: Wed Mar 20 15:56:06 2024 +0800
[SPARK-47473][SQL] Fix correctness issue of converting postgres INFINITY
timestamps
This PR fixes a bug involved with #41843 that Epoch Second is used instead
of epoch millis to create a timestamp value
bugfix
no
revised tests
no
Closes #45599 from yaooqinn/SPARK-47473.
Authored-by: Kent Yao <y...@apache.org>
Signed-off-by: Kent Yao <y...@apache.org>
(cherry picked from commit ad8ac17dbdfa763236ab3303eac6a3115ba710cc)
Signed-off-by: Kent Yao <y...@apache.org>
---
.../org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala | 9 ++++-----
.../main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala | 5 +++--
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
index 90d6f6ae2fbf..e910402e05e7 100644
---
a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
+++
b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
@@ -20,13 +20,13 @@ package org.apache.spark.sql.jdbc
import java.math.{BigDecimal => JBigDecimal}
import java.sql.{Connection, Date, Timestamp}
import java.text.SimpleDateFormat
-import java.time.{LocalDateTime, ZoneOffset}
+import java.time.LocalDateTime
import java.util.Properties
import org.apache.spark.sql.Column
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.types.{ArrayType, DecimalType, FloatType,
ShortType}
+import org.apache.spark.sql.types._
import org.apache.spark.tags.DockerTest
/**
@@ -445,9 +445,8 @@ class PostgresIntegrationSuite extends
DockerJDBCIntegrationSuite {
assert(row.length == 2)
val infinity = row(0).getAs[Timestamp]("timestamp_column")
val negativeInfinity = row(1).getAs[Timestamp]("timestamp_column")
- val minTimeStamp = LocalDateTime.of(1, 1, 1, 0, 0,
0).toEpochSecond(ZoneOffset.UTC)
- val maxTimestamp = LocalDateTime.of(9999, 12, 31, 23, 59,
59).toEpochSecond(ZoneOffset.UTC)
-
+ val minTimeStamp = -62135596800000L
+ val maxTimestamp = 253402300799999L
assert(infinity.getTime == maxTimestamp)
assert(negativeInfinity.getTime == minTimeStamp)
}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
index 55cd64fa898c..f8f72d88589e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
@@ -293,8 +293,9 @@ private object PostgresDialect extends JdbcDialect with
SQLConfHelper {
val POSTGRESQL_DATE_POSITIVE_INFINITY = 9223372036825200000L
val POSTGRESQL_DATE_DATE_POSITIVE_SMALLER_INFINITY = 185543533774800000L
- val minTimeStamp = LocalDateTime.of(1, 1, 1, 0, 0,
0).toEpochSecond(ZoneOffset.UTC)
- val maxTimestamp = LocalDateTime.of(9999, 12, 31, 23, 59,
59).toEpochSecond(ZoneOffset.UTC)
+ val minTimeStamp = LocalDateTime.of(1, 1, 1, 0, 0,
0).toInstant(ZoneOffset.UTC).toEpochMilli
+ val maxTimestamp =
+ LocalDateTime.of(9999, 12, 31, 23, 59, 59,
999999999).toInstant(ZoneOffset.UTC).toEpochMilli
val time = t.getTime
if (time == POSTGRESQL_DATE_POSITIVE_INFINITY ||
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
