yruslan commented on code in PR #41843:
URL: https://github.com/apache/spark/pull/41843#discussion_r1530575858
##########
sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala:
##########
@@ -281,4 +281,34 @@ private object PostgresDialect extends JdbcDialect with
SQLConfHelper {
}
s"ALTER TABLE ${getFullyQualifiedQuotedTableName(oldTable)} RENAME TO
${newTable.name()}"
}
+
+ /**
+ * java.sql timestamps are measured with millisecond accuracy (from
Long.MinValue
+ * milliseconds to Long.MaxValue milliseconds), while Spark timestamps are
measured
+ * at microseconds accuracy. For the "infinity values" in PostgreSQL
(represented by
+ * big constants), we need clamp them to avoid overflow. If it is not one of
the infinity
+ * values, fall back to default behavior.
+ */
+ override def convertJavaTimestampToTimestamp(t: Timestamp): Timestamp = {
+ // Variable names come from PostgreSQL "constant field docs":
+ //
https://jdbc.postgresql.org/documentation/publicapi/index.html?constant-values.html
+ val POSTGRESQL_DATE_NEGATIVE_INFINITY = -9223372036832400000L
+ val POSTGRESQL_DATE_NEGATIVE_SMALLER_INFINITY = -185543533774800000L
+ val POSTGRESQL_DATE_POSITIVE_INFINITY = 9223372036825200000L
+ val POSTGRESQL_DATE_DATE_POSITIVE_SMALLER_INFINITY = 185543533774800000L
+
+ val minTimeStamp = LocalDateTime.of(1, 1, 1, 0, 0,
0).toEpochSecond(ZoneOffset.UTC)
+ val maxTimestamp = LocalDateTime.of(9999, 12, 31, 23, 59,
59).toEpochSecond(ZoneOffset.UTC)
+
+ val time = t.getTime
Review Comment:
@yaooqinn, @mingkangli-db, @cloud-fan
`ts.getTime()` returns epoch in milliseconds, `new Timestamp(ts)` takes time
in **milliseconds** , and `LocalDateTime.toEpochSecond()` returns values in
**seconds**.
Shouldn't this be
```scala
val minTimeStamp = LocalDateTime.of(1, 1, 1, 0, 0,
0).toEpochSecond(ZoneOffset.UTC) * 1000
val maxTimestamp = LocalDateTime.of(9999, 12, 31, 23, 59,
59).toEpochSecond(ZoneOffset.UTC) * 1000
```
e.g. `* 1000`
Otherwise:
```scala
val tsLong = LocalDateTime.of(9999, 12, 31, 23, 59,
59).toEpochSecond(ZoneOffset.UTC)
val ts = new Timestamp(tsLong)
```
gives:
```
tsLong: Long = 253402300799
ts: java.sql.Timestamp = 1978-01-11 22:31:40.799
```
not `9999-12-31`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
