This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new c2ea6b34aa Implement min/max for interval types (#11015)
c2ea6b34aa is described below
commit c2ea6b34aaf603737f42753123662061b86c1ee5
Author: Max Burke <[email protected]>
AuthorDate: Sun Jun 23 03:17:30 2024 -0700
Implement min/max for interval types (#11015)
* Implement min/max for interval types
* Add sqllogictests for min/max intervals
* Add tests for interval min/max
* update sql logic tests
---------
Co-authored-by: Andrew Lamb <[email protected]>
---
datafusion/physical-expr/src/aggregate/min_max.rs | 139 +++++++++-
datafusion/sqllogictest/test_files/aggregate.slt | 307 +++++++++++-----------
2 files changed, 289 insertions(+), 157 deletions(-)
diff --git a/datafusion/physical-expr/src/aggregate/min_max.rs
b/datafusion/physical-expr/src/aggregate/min_max.rs
index a6d5054ec1..8d07f0df07 100644
--- a/datafusion/physical-expr/src/aggregate/min_max.rs
+++ b/datafusion/physical-expr/src/aggregate/min_max.rs
@@ -24,18 +24,20 @@ use
crate::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;
use crate::{AggregateExpr, PhysicalExpr};
use arrow::compute;
use arrow::datatypes::{
- DataType, Date32Type, Date64Type, Time32MillisecondType, Time32SecondType,
- Time64MicrosecondType, Time64NanosecondType, TimeUnit,
TimestampMicrosecondType,
- TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,
+ DataType, Date32Type, Date64Type, IntervalUnit, Time32MillisecondType,
+ Time32SecondType, Time64MicrosecondType, Time64NanosecondType, TimeUnit,
+ TimestampMicrosecondType, TimestampMillisecondType,
TimestampNanosecondType,
+ TimestampSecondType,
};
use arrow::{
array::{
ArrayRef, BinaryArray, BooleanArray, Date32Array, Date64Array,
Float32Array,
- Float64Array, Int16Array, Int32Array, Int64Array, Int8Array,
LargeBinaryArray,
- LargeStringArray, StringArray, Time32MillisecondArray,
Time32SecondArray,
- Time64MicrosecondArray, Time64NanosecondArray,
TimestampMicrosecondArray,
- TimestampMillisecondArray, TimestampNanosecondArray,
TimestampSecondArray,
- UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+ Float64Array, Int16Array, Int32Array, Int64Array, Int8Array,
+ IntervalDayTimeArray, IntervalMonthDayNanoArray,
IntervalYearMonthArray,
+ LargeBinaryArray, LargeStringArray, StringArray,
Time32MillisecondArray,
+ Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray,
+ TimestampMicrosecondArray, TimestampMillisecondArray,
TimestampNanosecondArray,
+ TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array,
UInt8Array,
},
datatypes::Field,
};
@@ -408,6 +410,25 @@ macro_rules! min_max_batch {
$OP
)
}
+ DataType::Interval(IntervalUnit::YearMonth) => {
+ typed_min_max_batch!(
+ $VALUES,
+ IntervalYearMonthArray,
+ IntervalYearMonth,
+ $OP
+ )
+ }
+ DataType::Interval(IntervalUnit::DayTime) => {
+ typed_min_max_batch!($VALUES, IntervalDayTimeArray,
IntervalDayTime, $OP)
+ }
+ DataType::Interval(IntervalUnit::MonthDayNano) => {
+ typed_min_max_batch!(
+ $VALUES,
+ IntervalMonthDayNanoArray,
+ IntervalMonthDayNano,
+ $OP
+ )
+ }
other => {
// This should have been handled before
return internal_err!(
@@ -1121,6 +1142,108 @@ impl Accumulator for SlidingMinAccumulator {
#[cfg(test)]
mod tests {
use super::*;
+ use arrow::datatypes::{
+ IntervalDayTimeType, IntervalMonthDayNanoType, IntervalYearMonthType,
+ };
+
+ #[test]
+ fn interval_min_max() {
+ // IntervalYearMonth
+ let b = IntervalYearMonthArray::from(vec![
+ IntervalYearMonthType::make_value(0, 1),
+ IntervalYearMonthType::make_value(5, 34),
+ IntervalYearMonthType::make_value(-2, 4),
+ IntervalYearMonthType::make_value(7, -4),
+ IntervalYearMonthType::make_value(0, 1),
+ ]);
+ let b: ArrayRef = Arc::new(b);
+
+ let mut min =
+
MinAccumulator::try_new(&DataType::Interval(IntervalUnit::YearMonth))
+ .unwrap();
+ min.update_batch(&[b.clone()]).unwrap();
+ let min_res = min.evaluate().unwrap();
+ assert_eq!(
+ min_res,
+
ScalarValue::IntervalYearMonth(Some(IntervalYearMonthType::make_value(
+ -2, 4
+ )))
+ );
+
+ let mut max =
+
MaxAccumulator::try_new(&DataType::Interval(IntervalUnit::YearMonth))
+ .unwrap();
+ max.update_batch(&[b.clone()]).unwrap();
+ let max_res = max.evaluate().unwrap();
+ assert_eq!(
+ max_res,
+
ScalarValue::IntervalYearMonth(Some(IntervalYearMonthType::make_value(
+ 5, 34
+ )))
+ );
+
+ // IntervalDayTime
+ let b = IntervalDayTimeArray::from(vec![
+ IntervalDayTimeType::make_value(0, 0),
+ IntervalDayTimeType::make_value(5, 454000),
+ IntervalDayTimeType::make_value(-34, 0),
+ IntervalDayTimeType::make_value(7, -4000),
+ IntervalDayTimeType::make_value(1, 0),
+ ]);
+ let b: ArrayRef = Arc::new(b);
+
+ let mut min =
+
MinAccumulator::try_new(&DataType::Interval(IntervalUnit::DayTime)).unwrap();
+ min.update_batch(&[b.clone()]).unwrap();
+ let min_res = min.evaluate().unwrap();
+ assert_eq!(
+ min_res,
+
ScalarValue::IntervalDayTime(Some(IntervalDayTimeType::make_value(-34, 0)))
+ );
+
+ let mut max =
+
MaxAccumulator::try_new(&DataType::Interval(IntervalUnit::DayTime)).unwrap();
+ max.update_batch(&[b.clone()]).unwrap();
+ let max_res = max.evaluate().unwrap();
+ assert_eq!(
+ max_res,
+
ScalarValue::IntervalDayTime(Some(IntervalDayTimeType::make_value(7, -4000)))
+ );
+
+ // IntervalMonthDayNano
+ let b = IntervalMonthDayNanoArray::from(vec![
+ IntervalMonthDayNanoType::make_value(1, 0, 0),
+ IntervalMonthDayNanoType::make_value(344, 34, -43_000_000_000),
+ IntervalMonthDayNanoType::make_value(-593, -33, 13_000_000_000),
+ IntervalMonthDayNanoType::make_value(5, 2, 493_000_000_000),
+ IntervalMonthDayNanoType::make_value(1, 0, 0),
+ ]);
+ let b: ArrayRef = Arc::new(b);
+
+ let mut min =
+
MinAccumulator::try_new(&DataType::Interval(IntervalUnit::MonthDayNano))
+ .unwrap();
+ min.update_batch(&[b.clone()]).unwrap();
+ let min_res = min.evaluate().unwrap();
+ assert_eq!(
+ min_res,
+ ScalarValue::IntervalMonthDayNano(Some(
+ IntervalMonthDayNanoType::make_value(-593, -33, 13_000_000_000)
+ ))
+ );
+
+ let mut max =
+
MaxAccumulator::try_new(&DataType::Interval(IntervalUnit::MonthDayNano))
+ .unwrap();
+ max.update_batch(&[b.clone()]).unwrap();
+ let max_res = max.evaluate().unwrap();
+ assert_eq!(
+ max_res,
+ ScalarValue::IntervalMonthDayNano(Some(
+ IntervalMonthDayNanoType::make_value(344, 34, -43_000_000_000)
+ ))
+ );
+ }
#[test]
fn float_min_max_with_nans() {
diff --git a/datafusion/sqllogictest/test_files/aggregate.slt
b/datafusion/sqllogictest/test_files/aggregate.slt
index 378cab2062..4090ff8764 100644
--- a/datafusion/sqllogictest/test_files/aggregate.slt
+++ b/datafusion/sqllogictest/test_files/aggregate.slt
@@ -1528,154 +1528,154 @@ e e 1323
query TTI
SELECT a.c1, b.c1, SUM(a.c2) FROM aggregate_test_100 as a CROSS JOIN
aggregate_test_100 as b GROUP BY CUBE (a.c1, b.c1) ORDER BY a.c1, b.c1
----
-a a 1260
-a b 1140
-a c 1260
-a d 1080
-a e 1260
-a NULL 6000
-b a 1302
-b b 1178
-b c 1302
-b d 1116
-b e 1302
-b NULL 6200
-c a 1176
-c b 1064
-c c 1176
-c d 1008
-c e 1176
-c NULL 5600
-d a 924
-d b 836
-d c 924
-d d 792
-d e 924
-d NULL 4400
-e a 1323
-e b 1197
-e c 1323
-e d 1134
-e e 1323
-e NULL 6300
-NULL a 5985
-NULL b 5415
-NULL c 5985
-NULL d 5130
-NULL e 5985
+a a 1260
+a b 1140
+a c 1260
+a d 1080
+a e 1260
+a NULL 6000
+b a 1302
+b b 1178
+b c 1302
+b d 1116
+b e 1302
+b NULL 6200
+c a 1176
+c b 1064
+c c 1176
+c d 1008
+c e 1176
+c NULL 5600
+d a 924
+d b 836
+d c 924
+d d 792
+d e 924
+d NULL 4400
+e a 1323
+e b 1197
+e c 1323
+e d 1134
+e e 1323
+e NULL 6300
+NULL a 5985
+NULL b 5415
+NULL c 5985
+NULL d 5130
+NULL e 5985
NULL NULL 28500
# csv_query_cube_distinct_count
query TII
SELECT c1, c2, COUNT(DISTINCT c3) FROM aggregate_test_100 GROUP BY CUBE
(c1,c2) ORDER BY c1,c2
----
-a 1 5
-a 2 3
-a 3 5
-a 4 4
-a 5 3
-a NULL 19
-b 1 3
-b 2 4
-b 3 2
-b 4 5
-b 5 5
-b NULL 17
-c 1 4
-c 2 7
-c 3 4
-c 4 4
-c 5 2
-c NULL 21
-d 1 7
-d 2 3
-d 3 3
-d 4 3
-d 5 2
-d NULL 18
-e 1 3
-e 2 4
-e 3 4
-e 4 7
-e 5 2
-e NULL 18
-NULL 1 22
-NULL 2 20
-NULL 3 17
-NULL 4 23
-NULL 5 14
+a 1 5
+a 2 3
+a 3 5
+a 4 4
+a 5 3
+a NULL 19
+b 1 3
+b 2 4
+b 3 2
+b 4 5
+b 5 5
+b NULL 17
+c 1 4
+c 2 7
+c 3 4
+c 4 4
+c 5 2
+c NULL 21
+d 1 7
+d 2 3
+d 3 3
+d 4 3
+d 5 2
+d NULL 18
+e 1 3
+e 2 4
+e 3 4
+e 4 7
+e 5 2
+e NULL 18
+NULL 1 22
+NULL 2 20
+NULL 3 17
+NULL 4 23
+NULL 5 14
NULL NULL 80
# csv_query_rollup_distinct_count
query TII
SELECT c1, c2, COUNT(DISTINCT c3) FROM aggregate_test_100 GROUP BY ROLLUP
(c1,c2) ORDER BY c1,c2
----
-a 1 5
-a 2 3
-a 3 5
-a 4 4
-a 5 3
-a NULL 19
-b 1 3
-b 2 4
-b 3 2
-b 4 5
-b 5 5
-b NULL 17
-c 1 4
-c 2 7
-c 3 4
-c 4 4
-c 5 2
-c NULL 21
-d 1 7
-d 2 3
-d 3 3
-d 4 3
-d 5 2
-d NULL 18
-e 1 3
-e 2 4
-e 3 4
-e 4 7
-e 5 2
-e NULL 18
+a 1 5
+a 2 3
+a 3 5
+a 4 4
+a 5 3
+a NULL 19
+b 1 3
+b 2 4
+b 3 2
+b 4 5
+b 5 5
+b NULL 17
+c 1 4
+c 2 7
+c 3 4
+c 4 4
+c 5 2
+c NULL 21
+d 1 7
+d 2 3
+d 3 3
+d 4 3
+d 5 2
+d NULL 18
+e 1 3
+e 2 4
+e 3 4
+e 4 7
+e 5 2
+e NULL 18
NULL NULL 80
# csv_query_rollup_sum_crossjoin
query TTI
SELECT a.c1, b.c1, SUM(a.c2) FROM aggregate_test_100 as a CROSS JOIN
aggregate_test_100 as b GROUP BY ROLLUP (a.c1, b.c1) ORDER BY a.c1, b.c1
----
-a a 1260
-a b 1140
-a c 1260
-a d 1080
-a e 1260
-a NULL 6000
-b a 1302
-b b 1178
-b c 1302
-b d 1116
-b e 1302
-b NULL 6200
-c a 1176
-c b 1064
-c c 1176
-c d 1008
-c e 1176
-c NULL 5600
-d a 924
-d b 836
-d c 924
-d d 792
-d e 924
-d NULL 4400
-e a 1323
-e b 1197
-e c 1323
-e d 1134
-e e 1323
-e NULL 6300
+a a 1260
+a b 1140
+a c 1260
+a d 1080
+a e 1260
+a NULL 6000
+b a 1302
+b b 1178
+b c 1302
+b d 1116
+b e 1302
+b NULL 6200
+c a 1176
+c b 1064
+c c 1176
+c d 1008
+c e 1176
+c NULL 5600
+d a 924
+d b 836
+d c 924
+d d 792
+d e 924
+d NULL 4400
+e a 1323
+e b 1197
+e c 1323
+e d 1134
+e e 1323
+e NULL 6300
NULL NULL 28500
# query_count_without_from
@@ -1785,30 +1785,39 @@ select min(t), max(t) from (select '00:00:00' as t
union select '00:00:01' unio
----
00:00:00 00:00:02
-# aggregate_decimal_min
-query RT
-select min(c1), arrow_typeof(min(c1)) from d_table
-----
--100.009 Decimal128(10, 3)
-
-# aggregate_decimal_max
-query RT
-select max(c1), arrow_typeof(max(c1)) from d_table
+# aggregate Interval(MonthDayNano) min/max
+query T??
+select
+ arrow_typeof(min(column1)), min(column1), max(column1)
+from values
+ (interval '1 month'),
+ (interval '2 months'),
+ (interval '2 month 15 days'),
+ (interval '-2 month')
----
-110.009 Decimal128(10, 3)
+Interval(MonthDayNano) 0 years -2 mons 0 days 0 hours 0 mins 0.000000000 secs
0 years 2 mons 15 days 0 hours 0 mins 0.000000000 secs
-# aggregate_decimal_sum
-query RT
-select sum(c1), arrow_typeof(sum(c1)) from d_table
+# aggregate Interval(DayTime) min/max
+query T??
+select
+ arrow_typeof(min(column1)), min(column1), max(column1)
+from values
+ (arrow_cast('60 minutes', 'Interval(DayTime)')),
+ (arrow_cast('-3 minutes', 'Interval(DayTime)')),
+ (arrow_cast('30 minutes', 'Interval(DayTime)'));
----
-100 Decimal128(20, 3)
+Interval(DayTime) 0 years 0 mons 0 days 0 hours -3 mins 0.000 secs 0 years 0
mons 0 days 1 hours 0 mins 0.000 secs
-# aggregate_decimal_avg
-query RT
-select avg(c1), arrow_typeof(avg(c1)) from d_table
+# aggregate Interval(YearMonth) min/max
+query T??
+select
+ arrow_typeof(min(column1)), min(column1), max(column1)
+from values
+ (arrow_cast('-1 year', 'Interval(YearMonth)')),
+ (arrow_cast('13 months', 'Interval(YearMonth)')),
+ (arrow_cast('1 year', 'Interval(YearMonth)'));
----
-5 Decimal128(14, 7)
-
+Interval(YearMonth) -1 years 0 mons 0 days 0 hours 0 mins 0.00 secs 1 years 1
mons 0 days 0 hours 0 mins 0.00 secs
# aggregate
query II
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
