This is an automated email from the ASF dual-hosted git repository. yao pushed a commit to branch branch-3.5 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push: new df2fddd5d008 [SPARK-47473][SQL] Fix correctness issue of converting postgres INFINITY timestamps df2fddd5d008 is described below commit df2fddd5d00860506534d5302f9b3f944d775deb Author: Kent Yao <y...@apache.org> AuthorDate: Wed Mar 20 15:56:06 2024 +0800 [SPARK-47473][SQL] Fix correctness issue of converting postgres INFINITY timestamps This PR fixes a bug involved with #41843 that Epoch Second is used instead of epoch millis to create a timestamp value bugfix no revised tests no Closes #45599 from yaooqinn/SPARK-47473. Authored-by: Kent Yao <y...@apache.org> Signed-off-by: Kent Yao <y...@apache.org> (cherry picked from commit ad8ac17dbdfa763236ab3303eac6a3115ba710cc) Signed-off-by: Kent Yao <y...@apache.org> --- .../org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala | 9 ++++----- .../main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala | 5 +++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala index 90d6f6ae2fbf..e910402e05e7 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala @@ -20,13 +20,13 @@ package org.apache.spark.sql.jdbc import java.math.{BigDecimal => JBigDecimal} import java.sql.{Connection, Date, Timestamp} import java.text.SimpleDateFormat -import java.time.{LocalDateTime, ZoneOffset} +import java.time.LocalDateTime import java.util.Properties import org.apache.spark.sql.Column import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.expressions.Literal -import org.apache.spark.sql.types.{ArrayType, DecimalType, FloatType, ShortType} +import org.apache.spark.sql.types._ import org.apache.spark.tags.DockerTest /** @@ -445,9 +445,8 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite { assert(row.length == 2) val infinity = row(0).getAs[Timestamp]("timestamp_column") val negativeInfinity = row(1).getAs[Timestamp]("timestamp_column") - val minTimeStamp = LocalDateTime.of(1, 1, 1, 0, 0, 0).toEpochSecond(ZoneOffset.UTC) - val maxTimestamp = LocalDateTime.of(9999, 12, 31, 23, 59, 59).toEpochSecond(ZoneOffset.UTC) - + val minTimeStamp = -62135596800000L + val maxTimestamp = 253402300799999L assert(infinity.getTime == maxTimestamp) assert(negativeInfinity.getTime == minTimeStamp) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala index 55cd64fa898c..f8f72d88589e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala @@ -293,8 +293,9 @@ private object PostgresDialect extends JdbcDialect with SQLConfHelper { val POSTGRESQL_DATE_POSITIVE_INFINITY = 9223372036825200000L val POSTGRESQL_DATE_DATE_POSITIVE_SMALLER_INFINITY = 185543533774800000L - val minTimeStamp = LocalDateTime.of(1, 1, 1, 0, 0, 0).toEpochSecond(ZoneOffset.UTC) - val maxTimestamp = LocalDateTime.of(9999, 12, 31, 23, 59, 59).toEpochSecond(ZoneOffset.UTC) + val minTimeStamp = LocalDateTime.of(1, 1, 1, 0, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli + val maxTimestamp = + LocalDateTime.of(9999, 12, 31, 23, 59, 59, 999999999).toInstant(ZoneOffset.UTC).toEpochMilli val time = t.getTime if (time == POSTGRESQL_DATE_POSITIVE_INFINITY || --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org