This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 439e94c [SPARK-35737][SQL] Parse day-time interval literals to
tightest types
439e94c is described below
commit 439e94c1712366ff267183d3946f2507ebf3a98e
Author: Kousuke Saruta <saru...@oss.nttdata.com>
AuthorDate: Mon Jun 14 10:06:19 2021 +0300
[SPARK-35737][SQL] Parse day-time interval literals to tightest types
### What changes were proposed in this pull request?
This PR add a feature which parse day-time interval literals to tightest
type.
### Why are the changes needed?
To comply with the ANSI behavior.
For example, `INTERVAL '10 20:30' DAY TO MINUTE` should be parsed as
`DayTimeIntervalType(DAY, MINUTE)` but not as `DayTimeIntervalType(DAY,
SECOND)`.
### Does this PR introduce _any_ user-facing change?
No because `DayTimeIntervalType` will be introduced in `3.2.0`.
### How was this patch tested?
New tests.
Closes #32892 from sarutak/tight-daytime-interval.
Authored-by: Kousuke Saruta <saru...@oss.nttdata.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
.../org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 9 +++++++--
.../resources/sql-tests/results/ansi/interval.sql.out | 12 ++++++------
.../src/test/resources/sql-tests/results/interval.sql.out | 12 ++++++------
.../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 15 +++++++++++++++
4 files changed, 34 insertions(+), 14 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 9f1d665..4bbd9bd 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -2357,9 +2357,14 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with
SQLConfHelper with Logg
Literal(calendarInterval.months, YearMonthIntervalType)
} else {
assert(calendarInterval.months == 0)
+ val strToFieldIndex = DayTimeIntervalType.dayTimeFields.map(i =>
+ DayTimeIntervalType.fieldToString(i) -> i).toMap
+ val fromUnit =
+
ctx.errorCapturingUnitToUnitInterval.body.from.getText.toLowerCase(Locale.ROOT)
val micros = IntervalUtils.getDuration(calendarInterval,
TimeUnit.MICROSECONDS)
- // TODO(SPARK-35737): Parse day-time interval literals to tightest
types
- Literal(micros, DayTimeIntervalType())
+ val start = strToFieldIndex(fromUnit)
+ val end = strToFieldIndex(toUnit)
+ Literal(micros, DayTimeIntervalType(start, end))
}
} else {
Literal(calendarInterval, CalendarIntervalType)
diff --git
a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index 3205259..f2f5d5c 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -363,7 +363,7 @@ struct<INTERVAL '10-9' YEAR TO MONTH:interval year to month>
-- !query
select interval '20 15' day to hour
-- !query schema
-struct<INTERVAL '20 15:00:00' DAY TO SECOND:interval day to second>
+struct<INTERVAL '20 15' DAY TO HOUR:interval day to hour>
-- !query output
20 15:00:00.000000000
@@ -371,7 +371,7 @@ struct<INTERVAL '20 15:00:00' DAY TO SECOND:interval day to
second>
-- !query
select interval '20 15:40' day to minute
-- !query schema
-struct<INTERVAL '20 15:40:00' DAY TO SECOND:interval day to second>
+struct<INTERVAL '20 15:40' DAY TO MINUTE:interval day to minute>
-- !query output
20 15:40:00.000000000
@@ -387,7 +387,7 @@ struct<INTERVAL '20 15:40:32.998999' DAY TO SECOND:interval
day to second>
-- !query
select interval '15:40' hour to minute
-- !query schema
-struct<INTERVAL '0 15:40:00' DAY TO SECOND:interval day to second>
+struct<INTERVAL '15:40' HOUR TO MINUTE:interval hour to minute>
-- !query output
0 15:40:00.000000000
@@ -395,7 +395,7 @@ struct<INTERVAL '0 15:40:00' DAY TO SECOND:interval day to
second>
-- !query
select interval '15:40:32.99899999' hour to second
-- !query schema
-struct<INTERVAL '0 15:40:32.998999' DAY TO SECOND:interval day to second>
+struct<INTERVAL '15:40:32.998999' HOUR TO SECOND:interval hour to second>
-- !query output
0 15:40:32.998999000
@@ -403,7 +403,7 @@ struct<INTERVAL '0 15:40:32.998999' DAY TO SECOND:interval
day to second>
-- !query
select interval '40:32.99899999' minute to second
-- !query schema
-struct<INTERVAL '0 00:40:32.998999' DAY TO SECOND:interval day to second>
+struct<INTERVAL '40:32.998999' MINUTE TO SECOND:interval minute to second>
-- !query output
0 00:40:32.998999000
@@ -411,7 +411,7 @@ struct<INTERVAL '0 00:40:32.998999' DAY TO SECOND:interval
day to second>
-- !query
select interval '40:32' minute to second
-- !query schema
-struct<INTERVAL '0 00:40:32' DAY TO SECOND:interval day to second>
+struct<INTERVAL '40:32' MINUTE TO SECOND:interval minute to second>
-- !query output
0 00:40:32.000000000
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index ef9ef8f..9b44960 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -357,7 +357,7 @@ struct<INTERVAL '10-9' YEAR TO MONTH:interval year to month>
-- !query
select interval '20 15' day to hour
-- !query schema
-struct<INTERVAL '20 15:00:00' DAY TO SECOND:interval day to second>
+struct<INTERVAL '20 15' DAY TO HOUR:interval day to hour>
-- !query output
20 15:00:00.000000000
@@ -365,7 +365,7 @@ struct<INTERVAL '20 15:00:00' DAY TO SECOND:interval day to
second>
-- !query
select interval '20 15:40' day to minute
-- !query schema
-struct<INTERVAL '20 15:40:00' DAY TO SECOND:interval day to second>
+struct<INTERVAL '20 15:40' DAY TO MINUTE:interval day to minute>
-- !query output
20 15:40:00.000000000
@@ -381,7 +381,7 @@ struct<INTERVAL '20 15:40:32.998999' DAY TO SECOND:interval
day to second>
-- !query
select interval '15:40' hour to minute
-- !query schema
-struct<INTERVAL '0 15:40:00' DAY TO SECOND:interval day to second>
+struct<INTERVAL '15:40' HOUR TO MINUTE:interval hour to minute>
-- !query output
0 15:40:00.000000000
@@ -389,7 +389,7 @@ struct<INTERVAL '0 15:40:00' DAY TO SECOND:interval day to
second>
-- !query
select interval '15:40:32.99899999' hour to second
-- !query schema
-struct<INTERVAL '0 15:40:32.998999' DAY TO SECOND:interval day to second>
+struct<INTERVAL '15:40:32.998999' HOUR TO SECOND:interval hour to second>
-- !query output
0 15:40:32.998999000
@@ -397,7 +397,7 @@ struct<INTERVAL '0 15:40:32.998999' DAY TO SECOND:interval
day to second>
-- !query
select interval '40:32.99899999' minute to second
-- !query schema
-struct<INTERVAL '0 00:40:32.998999' DAY TO SECOND:interval day to second>
+struct<INTERVAL '40:32.998999' MINUTE TO SECOND:interval minute to second>
-- !query output
0 00:40:32.998999000
@@ -405,7 +405,7 @@ struct<INTERVAL '0 00:40:32.998999' DAY TO SECOND:interval
day to second>
-- !query
select interval '40:32' minute to second
-- !query schema
-struct<INTERVAL '0 00:40:32' DAY TO SECOND:interval day to second>
+struct<INTERVAL '40:32' MINUTE TO SECOND:interval minute to second>
-- !query output
0 00:40:32.000000000
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 76c9f790d9..1cf5661 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -4003,6 +4003,21 @@ class SQLQuerySuite extends QueryTest with
SharedSparkSession with AdaptiveSpark
}
checkAnswer(sql(s"select /*+ REPARTITION(3, a) */ a b from values('123')
t(a)"), Row("123"))
}
+
+ test("SPARK-35737: Parse day-time interval literals to tightest types") {
+ val dayToSecDF = spark.sql("SELECT INTERVAL '13 02:02:10' DAY TO SECOND")
+ assert(dayToSecDF.schema.head.dataType === DayTimeIntervalType(0, 3))
+ val dayToMinuteDF = spark.sql("SELECT INTERVAL '-2 13:00' DAY TO MINUTE")
+ assert(dayToMinuteDF.schema.head.dataType === DayTimeIntervalType(0, 2))
+ val dayToHourDF = spark.sql("SELECT INTERVAL '0 15' DAY TO HOUR")
+ assert(dayToHourDF.schema.head.dataType === DayTimeIntervalType(0, 1))
+ val hourToSecDF = spark.sql("SELECT INTERVAL '00:21:02.03' HOUR TO SECOND")
+ assert(hourToSecDF.schema.head.dataType === DayTimeIntervalType(1, 3))
+ val hourToMinuteDF = spark.sql("SELECT INTERVAL '01:02' HOUR TO MINUTE")
+ assert(hourToMinuteDF.schema.head.dataType === DayTimeIntervalType(1, 2))
+ val minuteToSecDF = spark.sql("SELECT INTERVAL '10:03.775808000' MINUTE TO
SECOND")
+ assert(minuteToSecDF.schema.head.dataType === DayTimeIntervalType(2, 3))
+ }
}
case class Foo(bar: Option[String])
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
