sarutak commented on a change in pull request #32988:
URL: https://github.com/apache/spark/pull/32988#discussion_r655175054
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
##########
@@ -1115,34 +1116,47 @@ class DataFrameAggregateSuite extends QueryTest
}
test("SPARK-34716: Support ANSI SQL intervals by the aggregate function
`sum`") {
- val df = Seq((1, Period.ofMonths(10), Duration.ofDays(10)),
- (2, Period.ofMonths(1), Duration.ofDays(1)),
- (2, null, null),
- (3, Period.ofMonths(-3), Duration.ofDays(-6)),
- (3, Period.ofMonths(21), Duration.ofDays(-5)))
- .toDF("class", "year-month", "day-time")
+ val df = Seq(
+ (1, Period.ofMonths(10), Period.ofYears(8), Period.ofMonths(10),
Duration.ofDays(10)),
+ (2, Period.ofMonths(1), Period.ofYears(1), Period.ofMonths(1),
Duration.ofDays(1)),
+ (2, null, null, null, null),
+ (3, Period.ofMonths(-3), Period.ofYears(-12), Period.ofMonths(-3),
Duration.ofDays(-6)),
+ (3, Period.ofMonths(21), Period.ofYears(30), Period.ofMonths(5),
Duration.ofDays(-5)))
+ .toDF("class", "year-month", "year", "month", "day-time")
+ .select(
+ $"class",
+ $"year-month",
+ $"year" cast YearMonthIntervalType(YEAR, YEAR) as "year",
+ $"month" cast YearMonthIntervalType(MONTH, MONTH) as "month",
+ $"day-time")
val df2 = Seq((Period.ofMonths(Int.MaxValue), Duration.ofDays(106751991)),
(Period.ofMonths(10), Duration.ofDays(10)))
.toDF("year-month", "day-time")
- val sumDF = df.select(sum($"year-month"), sum($"day-time"))
- checkAnswer(sumDF, Row(Period.of(2, 5, 0), Duration.ofDays(0)))
+ val sumDF = df.select(sum($"year-month"), sum($"year"), sum($"month"),
sum($"day-time"))
+ checkAnswer(sumDF,
+ Row(Period.of(2, 5, 0), Period.ofYears(27), Period.of(1, 1, 0),
Duration.ofDays(0)))
assert(find(sumDF.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
assert(sumDF.schema == StructType(Seq(
- // TODO(SPARK-35775): Check all year-month interval types in aggregate
expressions
StructField("sum(year-month)", YearMonthIntervalType()),
+ StructField("sum(year)", YearMonthIntervalType(0, 0)),
+ StructField("sum(month)", YearMonthIntervalType(1, 1)),
Review comment:
Sorry, I've noticed after I pushed and already fixed.
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
##########
@@ -1115,34 +1116,47 @@ class DataFrameAggregateSuite extends QueryTest
}
test("SPARK-34716: Support ANSI SQL intervals by the aggregate function
`sum`") {
- val df = Seq((1, Period.ofMonths(10), Duration.ofDays(10)),
- (2, Period.ofMonths(1), Duration.ofDays(1)),
- (2, null, null),
- (3, Period.ofMonths(-3), Duration.ofDays(-6)),
- (3, Period.ofMonths(21), Duration.ofDays(-5)))
- .toDF("class", "year-month", "day-time")
+ val df = Seq(
+ (1, Period.ofMonths(10), Period.ofYears(8), Period.ofMonths(10),
Duration.ofDays(10)),
+ (2, Period.ofMonths(1), Period.ofYears(1), Period.ofMonths(1),
Duration.ofDays(1)),
+ (2, null, null, null, null),
+ (3, Period.ofMonths(-3), Period.ofYears(-12), Period.ofMonths(-3),
Duration.ofDays(-6)),
+ (3, Period.ofMonths(21), Period.ofYears(30), Period.ofMonths(5),
Duration.ofDays(-5)))
+ .toDF("class", "year-month", "year", "month", "day-time")
+ .select(
+ $"class",
+ $"year-month",
+ $"year" cast YearMonthIntervalType(YEAR, YEAR) as "year",
+ $"month" cast YearMonthIntervalType(MONTH, MONTH) as "month",
+ $"day-time")
val df2 = Seq((Period.ofMonths(Int.MaxValue), Duration.ofDays(106751991)),
(Period.ofMonths(10), Duration.ofDays(10)))
.toDF("year-month", "day-time")
- val sumDF = df.select(sum($"year-month"), sum($"day-time"))
- checkAnswer(sumDF, Row(Period.of(2, 5, 0), Duration.ofDays(0)))
+ val sumDF = df.select(sum($"year-month"), sum($"year"), sum($"month"),
sum($"day-time"))
+ checkAnswer(sumDF,
+ Row(Period.of(2, 5, 0), Period.ofYears(27), Period.of(1, 1, 0),
Duration.ofDays(0)))
assert(find(sumDF.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
assert(sumDF.schema == StructType(Seq(
- // TODO(SPARK-35775): Check all year-month interval types in aggregate
expressions
StructField("sum(year-month)", YearMonthIntervalType()),
+ StructField("sum(year)", YearMonthIntervalType(0, 0)),
+ StructField("sum(month)", YearMonthIntervalType(1, 1)),
Review comment:
Sorry, I've noticed right after I pushed and already fixed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
