This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 9004eb4e48 uncomment tests (#8881)
9004eb4e48 is described below
commit 9004eb4e4851a92e9bba60f0ca027514918ba1ae
Author: Alex Huang <[email protected]>
AuthorDate: Thu Jan 18 04:26:12 2024 +0800
uncomment tests (#8881)
---
datafusion/common/src/scalar.rs | 14 ++
datafusion/physical-expr/src/array_expressions.rs | 22 +--
datafusion/sqllogictest/test_files/array.slt | 166 +++++++++++-----------
3 files changed, 108 insertions(+), 94 deletions(-)
diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs
index 9cbd9e292f..20d03c7096 100644
--- a/datafusion/common/src/scalar.rs
+++ b/datafusion/common/src/scalar.rs
@@ -2957,6 +2957,20 @@ impl TryFrom<&DataType> for ScalarValue {
.to_owned()
.into(),
),
+ // 'ScalarValue::LargeList' contains single element `LargeListArray
+ DataType::LargeList(field) => ScalarValue::LargeList(
+ new_null_array(
+ &DataType::LargeList(Arc::new(Field::new(
+ "item",
+ field.data_type().clone(),
+ true,
+ ))),
+ 1,
+ )
+ .as_list::<i64>()
+ .to_owned()
+ .into(),
+ ),
DataType::Struct(fields) => ScalarValue::Struct(None,
fields.clone()),
DataType::Null => ScalarValue::Null,
_ => {
diff --git a/datafusion/physical-expr/src/array_expressions.rs
b/datafusion/physical-expr/src/array_expressions.rs
index 5b35c4b9d8..af6587631d 100644
--- a/datafusion/physical-expr/src/array_expressions.rs
+++ b/datafusion/physical-expr/src/array_expressions.rs
@@ -1074,7 +1074,9 @@ pub fn array_prepend(args: &[ArrayRef]) ->
Result<ArrayRef> {
}
}
-fn align_array_dimensions(args: Vec<ArrayRef>) -> Result<Vec<ArrayRef>> {
+fn align_array_dimensions<O: OffsetSizeTrait>(
+ args: Vec<ArrayRef>,
+) -> Result<Vec<ArrayRef>> {
let args_ndim = args
.iter()
.map(|arg| datafusion_common::utils::list_ndims(arg.data_type()))
@@ -1091,9 +1093,9 @@ fn align_array_dimensions(args: Vec<ArrayRef>) ->
Result<Vec<ArrayRef>> {
for _ in 0..(max_ndim - ndim) {
let data_type = aligned_array.data_type().to_owned();
let array_lengths = vec![1; aligned_array.len()];
- let offsets =
OffsetBuffer::<i32>::from_lengths(array_lengths);
+ let offsets =
OffsetBuffer::<O>::from_lengths(array_lengths);
- aligned_array = Arc::new(ListArray::try_new(
+ aligned_array = Arc::new(GenericListArray::<O>::try_new(
Arc::new(Field::new("item", data_type, true)),
offsets,
aligned_array,
@@ -1112,13 +1114,12 @@ fn align_array_dimensions(args: Vec<ArrayRef>) ->
Result<Vec<ArrayRef>> {
// Concatenate arrays on the same row.
fn concat_internal<O: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
- let args = align_array_dimensions(args.to_vec())?;
+ let args = align_array_dimensions::<O>(args.to_vec())?;
let list_arrays = args
.iter()
.map(|arg| as_generic_list_array::<O>(arg))
.collect::<Result<Vec<_>>>()?;
-
// Assume number of rows is the same for all arrays
let row_count = list_arrays[0].len();
@@ -2733,9 +2734,11 @@ mod tests {
let array2d_1 = Arc::new(array_into_list_array(array1d_1.clone())) as
ArrayRef;
let array2d_2 = Arc::new(array_into_list_array(array1d_2.clone())) as
ArrayRef;
- let res =
- align_array_dimensions(vec![array1d_1.to_owned(),
array2d_2.to_owned()])
- .unwrap();
+ let res = align_array_dimensions::<i32>(vec![
+ array1d_1.to_owned(),
+ array2d_2.to_owned(),
+ ])
+ .unwrap();
let expected = as_list_array(&array2d_1).unwrap();
let expected_dim =
datafusion_common::utils::list_ndims(array2d_1.data_type());
@@ -2748,7 +2751,8 @@ mod tests {
let array3d_1 = Arc::new(array_into_list_array(array2d_1)) as ArrayRef;
let array3d_2 = array_into_list_array(array2d_2.to_owned());
let res =
- align_array_dimensions(vec![array1d_1,
Arc::new(array3d_2.clone())]).unwrap();
+ align_array_dimensions::<i32>(vec![array1d_1,
Arc::new(array3d_2.clone())])
+ .unwrap();
let expected = as_list_array(&array3d_1).unwrap();
let expected_dim =
datafusion_common::utils::list_ndims(array3d_1.data_type());
diff --git a/datafusion/sqllogictest/test_files/array.slt
b/datafusion/sqllogictest/test_files/array.slt
index 6b45f204fe..342fcb5bec 100644
--- a/datafusion/sqllogictest/test_files/array.slt
+++ b/datafusion/sqllogictest/test_files/array.slt
@@ -107,18 +107,17 @@ AS VALUES
(make_array(make_array(4, 5, 6), make_array(10, 11, 12), make_array(4, 9,
8), make_array(7, 8, 9), make_array(10, 11, 12), make_array(1, 8, 7)),
make_array(10, 11, 12), 3, make_array([[11, 12, 13], [14, 15, 16]], [[17, 18,
19], [20, 21, 22]]), make_array(121, 131, 141))
;
-# TODO: add this when #8305 is fixed
-# statement ok
-# CREATE TABLE large_nested_arrays
-# AS
-# SELECT
-# arrow_cast(column1, 'LargeList(LargeList(Int64))') AS column1,
-# arrow_cast(column2, 'LargeList(Int64)') AS column2,
-# column3,
-# arrow_cast(column4, 'LargeList(LargeList(List(Int64)))') AS column4,
-# arrow_cast(column5, 'LargeList(Int64)') AS column5
-# FROM nested_arrays
-# ;
+statement ok
+CREATE TABLE large_nested_arrays
+AS
+ SELECT
+ arrow_cast(column1, 'LargeList(LargeList(Int64))') AS column1,
+ arrow_cast(column2, 'LargeList(Int64)') AS column2,
+ column3,
+ arrow_cast(column4, 'LargeList(LargeList(List(Int64)))') AS column4,
+ arrow_cast(column5, 'LargeList(Int64)') AS column5
+ FROM nested_arrays
+;
statement ok
CREATE TABLE arrays_values
@@ -155,16 +154,15 @@ AS VALUES
(NULL, NULL, NULL, NULL)
;
-# TODO: add this when #8305 is fixed
-# statement ok
-# CREATE TABLE large_arrays_values_v2
-# AS SELECT
-# arrow_cast(column1, 'LargeList(Int64)') AS column1,
-# arrow_cast(column2, 'LargeList(Int64)') AS column2,
-# column3,
-# arrow_cast(column4, 'LargeList(LargeList(Int64))') AS column4
-# FROM arrays_values_v2
-# ;
+statement ok
+CREATE TABLE large_arrays_values_v2
+AS SELECT
+ arrow_cast(column1, 'LargeList(Int64)') AS column1,
+ arrow_cast(column2, 'LargeList(Int64)') AS column2,
+ column3,
+ arrow_cast(column4, 'LargeList(LargeList(Int64))') AS column4
+FROM arrays_values_v2
+;
statement ok
CREATE TABLE flatten_table
@@ -1576,16 +1574,15 @@ select
----
[4] [] [1, , 3, 4] [, , 1]
-# TODO: add this when #8305 is fixed
-# query ????
-# select
-# array_append(arrow_cast(make_array(), 'LargeList(Null)'), 4),
-# array_append(make_array(), null),
-# array_append(make_array(1, null, 3), 4),
-# array_append(make_array(null, null), 1)
-# ;
-# ----
-# [4] [] [1, , 3, 4] [, , 1]
+query ????
+select
+ array_append(arrow_cast(make_array(), 'LargeList(Null)'), 4),
+ array_append(make_array(), null),
+ array_append(make_array(1, null, 3), 4),
+ array_append(make_array(null, null), 1)
+;
+----
+[4] [] [1, , 3, 4] [, , 1]
# test invalid (non-null)
query error
@@ -1604,13 +1601,12 @@ select
----
[[1, , 3], []] [[1, , 3], ]
-# TODO: add this when #8305 is fixed
-# query ??
-# select
-# array_append(arrow_cast(make_array(make_array(1, null, 3),
'LargeList(LargeList(Int64))')), arrow_cast(make_array(null),
'LargeList(Int64)')),
-# array_append(arrow_cast(make_array(make_array(1, null, 3),
'LargeList(LargeList(Int64))')), null);
-# ----
-# [[1, , 3], []] [[1, , 3], ]
+query ??
+select
+ array_append(arrow_cast(make_array(make_array(1, null, 3)),
'LargeList(LargeList(Int64))'), arrow_cast(make_array(null),
'LargeList(Int64)')),
+ array_append(arrow_cast(make_array(make_array(1, null, 3)),
'LargeList(LargeList(Int64))'), null);
+----
+[[1, , 3], []] [[1, , 3], ]
# array_append scalar function #3
query ???
@@ -1629,11 +1625,10 @@ select array_append(make_array([1], [2], [3]),
make_array(4)), array_append(make
----
[[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]]
-# TODO: add this when #8305 is fixed
-# query ???
-# select array_append(arrow_cast(make_array([1], [2], [3]),
'LargeList(LargeList(Int64))'), arrow_cast(make_array(4), 'LargeList(Int64)')),
array_append(arrow_cast(make_array([1.0], [2.0], [3.0]),
'LargeList(LargeList(Float64))'), arrow_cast(make_array(4.0),
'LargeList(Float64)')), array_append(arrow_cast(make_array(['h'], ['e'], ['l'],
['l']), 'LargeList(LargeList(Utf8))'), arrow_cast(make_array('o'),
'LargeList(Utf8)'));
-# ----
-# [[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]]
+query ???
+select array_append(arrow_cast(make_array([1], [2], [3]),
'LargeList(LargeList(Int64))'), arrow_cast(make_array(4), 'LargeList(Int64)')),
array_append(arrow_cast(make_array([1.0], [2.0], [3.0]),
'LargeList(LargeList(Float64))'), arrow_cast(make_array(4.0),
'LargeList(Float64)')), array_append(arrow_cast(make_array(['h'], ['e'], ['l'],
['l']), 'LargeList(LargeList(Utf8))'), arrow_cast(make_array('o'),
'LargeList(Utf8)'));
+----
+[[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]]
# list_append scalar function #5 (function alias `array_append`)
query ???
@@ -1700,12 +1695,11 @@ select array_append(column1, column2) from
nested_arrays;
[[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [7, 8, 9]]
[[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [10,
11, 12]]
-# TODO: add this when #8305 is fixed
-# query ?
-# select array_append(column1, column2) from large_nested_arrays;
-# ----
-# [[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [7, 8, 9]]
-# [[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7],
[10, 11, 12]]
+query ?
+select array_append(column1, column2) from large_nested_arrays;
+----
+[[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [7, 8, 9]]
+[[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [10,
11, 12]]
# array_append with columns and scalars #1
query ??
@@ -1737,12 +1731,11 @@ select array_append(column1, make_array(1, 11, 111)),
array_append(make_array(ma
[[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [1, 11,
111]] [[1, 2, 3], [11, 12, 13], [7, 8, 9]]
[[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [1,
11, 111]] [[1, 2, 3], [11, 12, 13], [10, 11, 12]]
-# TODO: add this when #8305 is fixed
-# query ??
-# select array_append(column1, arrow_cast(make_array(1, 11, 111),
'LargeList(Int64)')), array_append(arrow_cast(make_array(make_array(1, 2, 3),
make_array(11, 12, 13)), 'LargeList(LargeList(Int64))'), column2) from
large_nested_arrays;
-# ----
-# [[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [1, 11,
111]] [[1, 2, 3], [11, 12, 13], [7, 8, 9]]
-# [[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [1,
11, 111]] [[1, 2, 3], [11, 12, 13], [10, 11, 12]]
+query ??
+select array_append(column1, arrow_cast(make_array(1, 11, 111),
'LargeList(Int64)')), array_append(arrow_cast(make_array(make_array(1, 2, 3),
make_array(11, 12, 13)), 'LargeList(LargeList(Int64))'), column2) from
large_nested_arrays;
+----
+[[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [1, 11,
111]] [[1, 2, 3], [11, 12, 13], [7, 8, 9]]
+[[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [1,
11, 111]] [[1, 2, 3], [11, 12, 13], [10, 11, 12]]
## array_prepend (aliases: `list_prepend`, `array_push_front`,
`list_push_front`)
@@ -1816,11 +1809,12 @@ select array_prepend(make_array(1),
make_array(make_array(2), make_array(3), mak
----
[[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]]
-# TODO: add this when #8305 is fixed
-# query ???
-# select array_prepend(arrow_cast(make_array(1), 'LargeList(Int64)'),
arrow_cast(make_array(make_array(2), make_array(3), make_array(4)),
'LargeList(LargeList(Int64))')), array_prepend(arrow_cast(make_array(1.0),
'LargeList(Float64)'), arrow_cast(make_array([2.0], [3.0], [4.0]),
'LargeList(LargeList(Float64))')), array_prepend(arrow_cast(make_array('h'),
'LargeList(Utf8)'), arrow_cast(make_array(['e'], ['l'], ['l'], ['o']),
'LargeList(LargeList(Utf8))''));
-# ----
-# [[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]]
+query ???
+select array_prepend(arrow_cast(make_array(1), 'LargeList(Int64)'),
arrow_cast(make_array(make_array(2), make_array(3), make_array(4)),
'LargeList(LargeList(Int64))')),
+ array_prepend(arrow_cast(make_array(1.0), 'LargeList(Float64)'),
arrow_cast(make_array([2.0], [3.0], [4.0]), 'LargeList(LargeList(Float64))')),
+ array_prepend(arrow_cast(make_array('h'), 'LargeList(Utf8)'),
arrow_cast(make_array(['e'], ['l'], ['l'], ['o']),
'LargeList(LargeList(Utf8))'));
+----
+[[1], [2], [3], [4]] [[1.0], [2.0], [3.0], [4.0]] [[h], [e], [l], [l], [o]]
# list_prepend scalar function #5 (function alias `array_prepend`)
query ???
@@ -1887,12 +1881,11 @@ select array_prepend(column2, column1) from
nested_arrays;
[[7, 8, 9], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]]
[[10, 11, 12], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12],
[1, 8, 7]]
-# TODO: add this when #8305 is fixed
-# query ?
-# select array_prepend(column2, column1) from large_nested_arrays;
-# ----
-# [[7, 8, 9], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]]
-# [[10, 11, 12], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12],
[1, 8, 7]]
+query ?
+select array_prepend(column2, column1) from large_nested_arrays;
+----
+[[7, 8, 9], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]]
+[[10, 11, 12], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12],
[1, 8, 7]]
# array_prepend with columns and scalars #1
query ??
@@ -1924,12 +1917,11 @@ select array_prepend(make_array(1, 11, 111), column1),
array_prepend(column2, ma
[[1, 11, 111], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5,
6]] [[7, 8, 9], [1, 2, 3], [11, 12, 13]]
[[1, 11, 111], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12],
[1, 8, 7]] [[10, 11, 12], [1, 2, 3], [11, 12, 13]]
-# TODO: add this when #8305 is fixed
-# query ??
-# select array_prepend(arrow_cast(make_array(1, 11, 111), 'LargeList(Int64)'),
column1), array_prepend(column2, arrow_cast(make_array(make_array(1, 2, 3),
make_array(11, 12, 13)), 'LargeList(LargeList(Int64))')) from
large_nested_arrays;
-# ----
-# [[1, 11, 111], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5,
6]] [[7, 8, 9], [1, 2, 3], [11, 12, 13]]
-# [[1, 11, 111], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12],
[1, 8, 7]] [[10, 11, 12], [1, 2, 3], [11, 12, 13]]
+query ??
+select array_prepend(arrow_cast(make_array(1, 11, 111), 'LargeList(Int64)'),
column1), array_prepend(column2, arrow_cast(make_array(make_array(1, 2, 3),
make_array(11, 12, 13)), 'LargeList(LargeList(Int64))')) from
large_nested_arrays;
+----
+[[1, 11, 111], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5,
6]] [[7, 8, 9], [1, 2, 3], [11, 12, 13]]
+[[1, 11, 111], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12],
[1, 8, 7]] [[10, 11, 12], [1, 2, 3], [11, 12, 13]]
## array_repeat (aliases: `list_repeat`)
@@ -2372,12 +2364,11 @@ select array_position(column1, column2),
array_position(column1, column2, column
3 3
2 5
-#TODO: add this test when #8305 is fixed
-#query II
-#select array_position(column1, column2), array_position(column1, column2,
column3) from nested_arrays;
-#----
-#3 3
-#2 5
+query II
+select array_position(column1, column2), array_position(column1, column2,
column3) from nested_arrays;
+----
+3 3
+2 5
# array_position with columns and scalars #1
query III
@@ -2403,12 +2394,11 @@ select array_position(make_array([1, 2, 3], [4, 5, 6],
[11, 12, 13]), column2),
NULL 6 4
NULL 1 NULL
-#TODO: add this test when #8305 is fixed
-#query III
-#select array_position(arrow_cast(make_array([1, 2, 3], [4, 5, 6], [11, 12,
13]), 'LargeList(List(Int64))'), column2), array_position(column1,
make_array(4, 5, 6)), array_position(column1, make_array(1, 2, 3), 2) from
large_nested_arrays;
-#----
-#NULL 6 4
-#NULL 1 NULL
+query III
+select array_position(arrow_cast(make_array([1, 2, 3], [4, 5, 6], [11, 12,
13]), 'LargeList(LargeList(Int64))'), column2), array_position(column1,
arrow_cast(make_array(4, 5, 6), 'LargeList(Int64)')), array_position(column1,
arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'), 2) from
large_nested_arrays;
+----
+NULL 6 4
+NULL 1 NULL
## array_positions (aliases: `list_positions`)
@@ -4989,6 +4979,9 @@ drop table values_without_nulls;
statement ok
drop table nested_arrays;
+statement ok
+drop table large_nested_arrays;
+
statement ok
drop table arrays;
@@ -5007,6 +5000,9 @@ drop table arrays_values;
statement ok
drop table arrays_values_v2;
+statement ok
+drop table large_arrays_values_v2;
+
statement ok
drop table array_has_table_1D;
