This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new fe18354 [SPARK-35679][SQL] instantToMicros overflow
fe18354 is described below
commit fe1835406ac46eca2b147e59a92a91ba04f61121
Author: dgd-contributor <[email protected]>
AuthorDate: Thu Jun 10 08:08:51 2021 +0300
[SPARK-35679][SQL] instantToMicros overflow
With Long.minValue cast to an instant, secs will be floored in function
microsToInstant and cause overflow when multiply with Micros_per_second
```
def microsToInstant(micros: Long): Instant = {
val secs = Math.floorDiv(micros, MICROS_PER_SECOND)
// Unfolded Math.floorMod(us, MICROS_PER_SECOND) to reuse the result of
// the above calculation of `secs` via `floorDiv`.
val mos = micros - secs * MICROS_PER_SECOND <- it will overflow here
Instant.ofEpochSecond(secs, mos * NANOS_PER_MICROS)
}
```
But the overflow is acceptable because it won't produce any change to the
result
However, when convert the instant back to micro value, it will raise
Overflow Error
```
def instantToMicros(instant: Instant): Long = {
val us = Math.multiplyExact(instant.getEpochSecond, MICROS_PER_SECOND) <-
It overflow here
val result = Math.addExact(us, NANOSECONDS.toMicros(instant.getNano))
result
}
```
Code to reproduce this error
```
instantToMicros(microToInstant(Long.MinValue))
```
No
Test added
Closes #32839 from dgd-contributor/SPARK-35679_instantToMicro.
Authored-by: dgd-contributor <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
(cherry picked from commit aa3de4077302fe7e0b23b01a338c7feab0e5974e)
Signed-off-by: Max Gekk <[email protected]>
---
.../spark/sql/catalyst/util/DateTimeUtils.scala | 20 +++++++++++++++++---
.../spark/sql/catalyst/util/DateTimeUtilsSuite.scala | 5 +++++
2 files changed, 22 insertions(+), 3 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 52df227..2e242f2 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -413,10 +413,24 @@ object DateTimeUtils {
}
}
+ // See issue SPARK-35679
+ // min second cause overflow in instant to micro
+ private val MIN_SECONDS = Math.floorDiv(Long.MinValue, MICROS_PER_SECOND)
+
+ /**
+ * Gets the number of microseconds since the epoch of 1970-01-01 00:00:00Z
from the given
+ * instance of `java.time.Instant`. The epoch microsecond count is a simple
incrementing count of
+ * microseconds where microsecond 0 is 1970-01-01 00:00:00Z.
+ */
def instantToMicros(instant: Instant): Long = {
- val us = Math.multiplyExact(instant.getEpochSecond, MICROS_PER_SECOND)
- val result = Math.addExact(us, NANOSECONDS.toMicros(instant.getNano))
- result
+ val secs = instant.getEpochSecond
+ if (secs == MIN_SECONDS) {
+ val us = Math.multiplyExact(secs + 1, MICROS_PER_SECOND)
+ Math.addExact(us, NANOSECONDS.toMicros(instant.getNano) -
MICROS_PER_SECOND)
+ } else {
+ val us = Math.multiplyExact(secs, MICROS_PER_SECOND)
+ Math.addExact(us, NANOSECONDS.toMicros(instant.getNano))
+ }
}
def microsToInstant(us: Long): Instant = {
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index d457c88..d451aff 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -686,4 +686,9 @@ class DateTimeUtilsSuite extends SparkFunSuite with
Matchers with SQLHelper {
assert(toDate("tomorrow CET ", zoneId).get === today + 1)
}
}
+
+ test("SPARK-35679: instantToMicros should be able to return microseconds of
Long.MinValue") {
+ assert(instantToMicros(microsToInstant(Long.MaxValue)) === Long.MaxValue)
+ assert(instantToMicros(microsToInstant(Long.MinValue)) === Long.MinValue)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
