This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 072e0eb [SPARK-28308][CORE] CalendarInterval sub-second part should
be padded before parsing
072e0eb is described below
commit 072e0eb8881be7df7d5c81efa472b448b9d67e95
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Mon Jul 8 19:40:41 2019 -0700
[SPARK-28308][CORE] CalendarInterval sub-second part should be padded
before parsing
The sub-second part of the interval should be padded before parsing.
Currently, Spark gives a correct value only when there is 9 digits below `.`.
```
spark-sql> select interval '0 0:0:0.123456789' day to second;
interval 123 milliseconds 456 microseconds
spark-sql> select interval '0 0:0:0.12345678' day to second;
interval 12 milliseconds 345 microseconds
spark-sql> select interval '0 0:0:0.1234' day to second;
interval 1 microseconds
```
Pass the Jenkins with the fixed test cases.
Closes #25079 from dongjoon-hyun/SPARK-28308.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit a5ff9221fc23cc758db228493d451f542591eff7)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java | 3 ++-
.../java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java | 3 ++-
.../scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 4 +++-
3 files changed, 7 insertions(+), 3 deletions(-)
diff --git
a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
index e36efa3..1818fef 100644
---
a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
+++
b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
@@ -177,7 +177,8 @@ public final class CalendarInterval implements Serializable
{
long minutes = toLongWithRange("minute", m.group(4), 0, 59);
long seconds = toLongWithRange("second", m.group(5), 0, 59);
// Hive allow nanosecond precision interval
- long nanos = toLongWithRange("nanosecond", m.group(7), 0L, 999999999L);
+ String nanoStr = m.group(7) == null ? null : (m.group(7) +
"000000000").substring(0, 9);
+ long nanos = toLongWithRange("nanosecond", nanoStr, 0L, 999999999L);
result = new CalendarInterval(0, sign * (
days * MICROS_PER_DAY + hours * MICROS_PER_HOUR + minutes *
MICROS_PER_MINUTE +
seconds * MICROS_PER_SECOND + nanos / 1000L));
diff --git
a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java
b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java
index 994af8f..c125ba5 100644
---
a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java
+++
b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java
@@ -162,7 +162,8 @@ public class CalendarIntervalSuite {
assertEquals(fromDayTimeString(input), i);
input = "10 0:12:0.888";
- i = new CalendarInterval(0, 10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE);
+ i = new CalendarInterval(0, 10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE +
+ 888 * MICROS_PER_MILLI);
assertEquals(fromDayTimeString(input), i);
input = "-3 0:0:0";
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 2e5e5c0..17cbb15 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -1178,11 +1178,13 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils
with TestHiveSingleton {
}
test("Convert hive interval term into Literal of CalendarIntervalType") {
+ checkAnswer(sql("select interval '0 0:0:0.1' day to second"),
+ Row(CalendarInterval.fromString("interval 100 milliseconds")))
checkAnswer(sql("select interval '10-9' year to month"),
Row(CalendarInterval.fromString("interval 10 years 9 months")))
checkAnswer(sql("select interval '20 15:40:32.99899999' day to second"),
Row(CalendarInterval.fromString("interval 2 weeks 6 days 15 hours 40
minutes " +
- "32 seconds 99 milliseconds 899 microseconds")))
+ "32 seconds 998 milliseconds 999 microseconds")))
checkAnswer(sql("select interval '30' year"),
Row(CalendarInterval.fromString("interval 30 years")))
checkAnswer(sql("select interval '25' month"),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
